├── .github └── workflows │ └── build.yml ├── .gitignore ├── .idea ├── .gitignore ├── duckdb_protobuf.iml ├── modules.xml ├── runConfigurations │ ├── build_debug.xml │ └── build_release.xml └── vcs.xml ├── Cargo.lock ├── Cargo.toml ├── Makefile ├── README.md ├── flake.lock ├── flake.nix ├── packages ├── duckdb_metadata │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── duckdb_metadata_bin │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── duckdb_protobuf │ ├── Cargo.toml │ ├── src │ │ ├── filtered_dynamic_message.rs │ │ ├── io.rs │ │ ├── lib.rs │ │ ├── read.rs │ │ ├── types.rs │ │ └── vtab.rs │ └── tests │ │ ├── .gitignore │ │ ├── it │ │ └── main.rs │ │ ├── protos │ │ └── user.proto │ │ └── src │ │ └── user.rs └── serving │ ├── .gitignore │ ├── package.json │ ├── src │ └── worker.ts │ ├── tsconfig.json │ ├── wrangler.toml │ └── yarn.lock └── patches ├── duckdb+1.0.0.patch ├── duckdb-loadable-macros+0.1.2.patch └── libduckdb-sys+1.0.0.patch /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_dispatch: 3 | push: 4 | tags: 5 | - '*' 6 | branches: 7 | - main 8 | - master 9 | 10 | jobs: 11 | build: 12 | name: build ${{ matrix.platform }} ${{ matrix.target }} 13 | strategy: 14 | matrix: 15 | include: 16 | - platform: windows-latest 17 | target: x86_64-pc-windows-msvc 18 | duckdb_platform: windows_amd64 19 | library_output: duckdb_protobuf.dll 20 | - platform: macos-latest 21 | target: aarch64-apple-darwin 22 | library_output: libduckdb_protobuf.dylib 23 | duckdb_platform: osx_arm64 24 | - platform: ubuntu-latest 25 | target: x86_64-unknown-linux-gnu 26 | library_output: libduckdb_protobuf.so 27 | duckdb_platform: linux_amd64_gcc4 28 | 29 | runs-on: ${{ matrix.platform }} 30 | 31 | steps: 32 | - uses: actions/checkout@v4 33 | 34 | - run: make vendor 35 | 36 | - run: cargo build --target ${{ matrix.target }} --release 37 | 38 | - name: attach metadata 39 | run: > 40 | cargo run 41 | --package duckdb_metadata_bin 42 | --bin duckdb_metadata 43 | -- 44 | --input target/${{ matrix.target }}/release/${{ matrix.library_output }} 45 | --output target/${{ matrix.target }}/release/protobuf.duckdb_extension 46 | --extension-version v0.0.1 47 | --duckdb-api-version v0.0.1 48 | --extension-abi-type C_STRUCT 49 | --platform ${{ matrix.duckdb_platform }} 50 | 51 | - name: save artifacts 52 | if: success() 53 | uses: actions/upload-artifact@v4 54 | with: 55 | name: build-${{ matrix.platform }}-${{ matrix.target }} 56 | path: target/${{ matrix.target }}/release/protobuf.duckdb_extension 57 | 58 | release: 59 | needs: build 60 | runs-on: ubuntu-latest 61 | if: startsWith(github.ref, 'refs/tags/') 62 | steps: 63 | - uses: actions/download-artifact@v4 64 | 65 | - run: | 66 | mkdir release 67 | mv build-macos-latest-aarch64-apple-darwin/protobuf.duckdb_extension ./release/protobuf.osx_arm64.duckdb_extension 68 | mv build-ubuntu-latest-x86_64-unknown-linux-gnu/protobuf.duckdb_extension ./release/protobuf.linux_amd64_gcc4.duckdb_extension 69 | mv build-windows-latest-x86_64-pc-windows-msvc/protobuf.duckdb_extension ./release/protobuf.windows_amd64.duckdb_extension 70 | 71 | - name: upload files to r2 72 | env: 73 | AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} 74 | AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} 75 | AWS_DEFAULT_REGION: auto 76 | run: | 77 | aws s3 cp \ 78 | --endpoint-url https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com \ 79 | ./release/protobuf.osx_arm64.duckdb_extension \ 80 | s3://duckdb-extensions/duckdb-api-version/v0.0.1/osx_arm64/protobuf.duckdb_extension 81 | 82 | aws s3 cp \ 83 | --endpoint-url https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com \ 84 | ./release/protobuf.linux_amd64_gcc4.duckdb_extension \ 85 | s3://duckdb-extensions/duckdb-api-version/v0.0.1/linux_amd64_gcc4/protobuf.duckdb_extension 86 | 87 | aws s3 cp \ 88 | --endpoint-url https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com \ 89 | ./release/protobuf.windows_amd64.duckdb_extension \ 90 | s3://duckdb-extensions/duckdb-api-version/v0.0.1/windows_amd64/protobuf.duckdb_extension 91 | 92 | - name: release 93 | uses: softprops/action-gh-release@v2 94 | with: 95 | files: release/* 96 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | 3 | packages/vendor 4 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | -------------------------------------------------------------------------------- /.idea/duckdb_protobuf.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/runConfigurations/build_debug.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/runConfigurations/build_release.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | 4 | members = [ 5 | "packages/vendor/duckdb", 6 | "packages/vendor/duckdb-loadable-macros", 7 | "packages/vendor/libduckdb-sys", 8 | "packages/duckdb_metadata", 9 | "packages/duckdb_metadata_bin", 10 | "packages/duckdb_protobuf" 11 | ] 12 | 13 | [patch.crates-io] 14 | duckdb = { path = 'packages/vendor/duckdb' } 15 | duckdb-loadable-macros = { path = 'packages/vendor/duckdb-loadable-macros' } 16 | libduckdb-sys = { path = 'packages/vendor/libduckdb-sys' } 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # A Makefile for building stuff during development. This is not used for the CI 2 | # builds. These two are separate concepts with different constraints (which 3 | # target is specified, cross compilation, etc.). Using this as a task runner (a 4 | # la just). 5 | 6 | DUCKDB_PLATFORM := osx_arm64 7 | DUCKDB_EXTENSION_VERSION := v0.0.1 8 | DUCKDB_API_VERSION := v0.0.1 9 | 10 | ifeq ($(DUCKDB_PLATFORM),windows_amd64) 11 | LIBRARY_OUTPUT := duckdb_protobuf.dll 12 | endif 13 | ifeq ($(DUCKDB_PLATFORM),osx_arm64) 14 | LIBRARY_OUTPUT := libduckdb_protobuf.dylib 15 | endif 16 | ifeq ($(DUCKDB_PLATFORM),linux_amd64) 17 | LIBRARY_OUTPUT := libduckdb_protobuf.so 18 | endif 19 | 20 | packages/vendor/duckdb: 21 | mkdir -p packages/vendor/duckdb 22 | curl -L https://crates.io/api/v1/crates/duckdb/1.0.0/download | tar --strip-components=1 -xz -C packages/vendor/duckdb 23 | patch --strip=1 --directory=packages/vendor/duckdb < patches/duckdb+1.0.0.patch 24 | 25 | packages/vendor/duckdb-loadable-macros: 26 | mkdir -p packages/vendor/duckdb-loadable-macros 27 | curl -L https://crates.io/api/v1/crates/duckdb-loadable-macros/0.1.2/download | tar --strip-components=1 -xz -C packages/vendor/duckdb-loadable-macros 28 | patch --strip=1 --directory=packages/vendor/duckdb-loadable-macros < patches/duckdb-loadable-macros+0.1.2.patch 29 | 30 | packages/vendor/libduckdb-sys: 31 | mkdir -p packages/vendor/libduckdb-sys 32 | curl -L https://crates.io/api/v1/crates/libduckdb-sys/1.0.0/download | tar --strip-components=1 -xz -C packages/vendor/libduckdb-sys 33 | patch --strip=1 --directory=packages/vendor/libduckdb-sys < patches/libduckdb-sys+1.0.0.patch 34 | 35 | vendor: packages/vendor/duckdb packages/vendor/duckdb-loadable-macros packages/vendor/libduckdb-sys 36 | 37 | debug: vendor 38 | cargo build --package duckdb_protobuf 39 | cargo run \ 40 | --package duckdb_metadata_bin \ 41 | --bin duckdb_metadata \ 42 | -- \ 43 | --input target/debug/$(LIBRARY_OUTPUT) \ 44 | --output target/debug/protobuf.duckdb_extension \ 45 | --extension-version $(DUCKDB_EXTENSION_VERSION) \ 46 | --duckdb-api-version $(DUCKDB_API_VERSION) \ 47 | --platform $(DUCKDB_PLATFORM) \ 48 | --extension-abi-type C_STRUCT 49 | 50 | release: vendor 51 | cargo build --package duckdb_protobuf --release 52 | cargo run \ 53 | --package duckdb_metadata_bin \ 54 | --bin duckdb_metadata \ 55 | -- \ 56 | --input target/release/$(LIBRARY_OUTPUT) \ 57 | --output target/release/protobuf.duckdb_extension \ 58 | --extension-version $(DUCKDB_EXTENSION_VERSION) \ 59 | --duckdb-api-version $(DUCKDB_API_VERSION) \ 60 | --platform $(DUCKDB_PLATFORM) \ 61 | --extension-abi-type C_STRUCT 62 | 63 | test: release 64 | cargo test --package duckdb_protobuf 65 | 66 | install: release 67 | duckdb \ 68 | -unsigned \ 69 | -cmd "FORCE INSTALL 'target/release/protobuf.duckdb_extension'" \ 70 | -no-stdin 71 | 72 | .PHONY: test release debug vendor -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # duckdb_protobuf 2 | 3 | a duckdb extension for parsing sequences of protobuf messages encoded in either 4 | the standard varint delimited format or a u32 big endian delimited format. 5 | 6 | ## quick start 7 | 8 | ensure you're using duckdb 1.1.0 for support with the latest features. if you 9 | need new features on an old versions, please open an issue. 10 | 11 | ```bash 12 | $ duckdb -version 13 | v1.1.0 fa5c2fe15f 14 | ``` 15 | 16 | start duckdb with `-unsigned` flag to allow loading unsigned libraries. 17 | 18 | ```bash 19 | $ duckdb -unsigned 20 | ``` 21 | 22 | or if you're using the jdbc connector, you can do this with the 23 | `allow_unsigned_extensions` jdbc connection property. 24 | 25 | now install the extension: 26 | 27 | ```sql 28 | INSTALL protobuf from 'https://duckdb.0xcaff.xyz'; 29 | ``` 30 | 31 | next load it (you'll need to do this once for every session you want to use 32 | the extension) 33 | 34 | ```sql 35 | LOAD protobuf; 36 | ``` 37 | 38 | and start shredding up your protobufs! 39 | 40 | ```sql 41 | SELECT * 42 | FROM protobuf( 43 | descriptors = './descriptor.pb', 44 | files = './scrape/data/SceneVersion/**/*.bin', 45 | message_type = 'test_server.v1.GetUserSceneVersionResponse', 46 | delimiter = 'BigEndianFixed' 47 | ) 48 | LIMIT 10; 49 | ``` 50 | 51 | if you want builds for a platform or version which currently doesn't have 52 | builds, please open an issue. 53 | 54 |
55 | install from file 56 | 57 | download the latest version from 58 | [releases](https://github.com/0xcaff/duckdb_protobuf/releases). if you're on 59 | macOS, blow away the quarantine params with the following to allow the file 60 | to be loaded 61 | 62 | ```sh 63 | $ xattr -d com.apple.quarantine /Users/martin/Downloads/protobuf.duckdb_extension 64 | ``` 65 | 66 | next load the extension 67 | 68 | ```sql 69 | LOAD '/Users/martin/Downloads/protobuf.duckdb_extension'; 70 | ``` 71 |
72 | 73 | ## why 74 | 75 | sometimes you want to land your row primary data in a format with a well-defined 76 | structure and pretty good decode performance and poke around without a load 77 | step. maybe you're scraping an endpoint which returns protobuf responses, you're 78 | figuring out the schema as you go and iteration speed matters much more than 79 | query performance. 80 | 81 | `duckdb_protobuf` allows for making a new choice along the 82 | flexibility-performance tradeoff continuum for fast exploration of protobuf 83 | streams with little upfront load complexity or time. 84 | 85 | ## configuration 86 | 87 | * `descriptors`: path to the protobuf descriptor file. Generated using something 88 | like `protoc --descriptor_set_out=descriptor.pb ...` 89 | * `files`: glob pattern for the files to read. Uses the [`glob`][glob] crate 90 | for evaluating globs. 91 | * `message_type`: the fully qualified message type to parse. 92 | * `delimiter`: specifies where one message starts and the next one begins 93 | * `BigEndianFixed`: every message is prefixed with a u32 big endian value 94 | specifying its length. files are a sequence of messages 95 | * `Varint`: every message is prefixed with a protobuf Varint value 96 | ([encoding](https://protobuf.dev/programming-guides/encoding/#varints)). 97 | files are a sequence of messages 98 | * `SingleMessagePerFile`: each file contains a single message 99 | * `filename`, `position` and `size`: boolean values enabling columns which add 100 | source information about where the messages originated from 101 | 102 | ## features 103 | 104 | * converts `google.protobuf.Timestamp` messages to duckdb timestamp 105 | * supports nested messages with repeating fields 106 | * scales decoding across as many threads as duckdb allows 107 | * supports projection pushdown (for first level of columns) ensuring only 108 | necessary columns are decoded. 109 | 110 | ## limitations 111 | 112 | * doesn't support a few types (bytes, maps, {s,}fixed{32,64}, sint{32,64}), 113 | contributions and even feedback that these field types are used is welcome! 114 | 115 | i'm releasing this to understand how other folks are using protobuf streams and 116 | duckdb. i'm open to PRs, issues and other feedback. 117 | 118 | [glob]: https://docs.rs/glob/latest/glob/ -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "cachix": { 4 | "inputs": { 5 | "devenv": [ 6 | "crate2nix" 7 | ], 8 | "flake-compat": [ 9 | "crate2nix" 10 | ], 11 | "nixpkgs": "nixpkgs", 12 | "pre-commit-hooks": [ 13 | "crate2nix" 14 | ] 15 | }, 16 | "locked": { 17 | "lastModified": 1709700175, 18 | "narHash": "sha256-A0/6ZjLmT9qdYzKHmevnEIC7G+GiZ4UCr8v0poRPzds=", 19 | "owner": "cachix", 20 | "repo": "cachix", 21 | "rev": "be97b37989f11b724197b5f4c7ffd78f12c8c4bf", 22 | "type": "github" 23 | }, 24 | "original": { 25 | "owner": "cachix", 26 | "ref": "latest", 27 | "repo": "cachix", 28 | "type": "github" 29 | } 30 | }, 31 | "cachix_2": { 32 | "inputs": { 33 | "devenv": [ 34 | "crate2nix", 35 | "crate2nix_stable" 36 | ], 37 | "flake-compat": [ 38 | "crate2nix", 39 | "crate2nix_stable" 40 | ], 41 | "nixpkgs": "nixpkgs_2", 42 | "pre-commit-hooks": [ 43 | "crate2nix", 44 | "crate2nix_stable" 45 | ] 46 | }, 47 | "locked": { 48 | "lastModified": 1716549461, 49 | "narHash": "sha256-lHy5kgx6J8uD+16SO47dPrbob98sh+W1tf4ceSqPVK4=", 50 | "owner": "cachix", 51 | "repo": "cachix", 52 | "rev": "e2bb269fb8c0828d5d4d2d7b8d09ea85abcacbd4", 53 | "type": "github" 54 | }, 55 | "original": { 56 | "owner": "cachix", 57 | "ref": "latest", 58 | "repo": "cachix", 59 | "type": "github" 60 | } 61 | }, 62 | "cachix_3": { 63 | "inputs": { 64 | "devenv": [ 65 | "crate2nix", 66 | "crate2nix_stable", 67 | "crate2nix_stable" 68 | ], 69 | "flake-compat": [ 70 | "crate2nix", 71 | "crate2nix_stable", 72 | "crate2nix_stable" 73 | ], 74 | "nixpkgs": "nixpkgs_3", 75 | "pre-commit-hooks": [ 76 | "crate2nix", 77 | "crate2nix_stable", 78 | "crate2nix_stable" 79 | ] 80 | }, 81 | "locked": { 82 | "lastModified": 1716549461, 83 | "narHash": "sha256-lHy5kgx6J8uD+16SO47dPrbob98sh+W1tf4ceSqPVK4=", 84 | "owner": "cachix", 85 | "repo": "cachix", 86 | "rev": "e2bb269fb8c0828d5d4d2d7b8d09ea85abcacbd4", 87 | "type": "github" 88 | }, 89 | "original": { 90 | "owner": "cachix", 91 | "ref": "latest", 92 | "repo": "cachix", 93 | "type": "github" 94 | } 95 | }, 96 | "crate2nix": { 97 | "inputs": { 98 | "cachix": "cachix", 99 | "crate2nix_stable": "crate2nix_stable", 100 | "devshell": "devshell_3", 101 | "flake-compat": "flake-compat_3", 102 | "flake-parts": "flake-parts_3", 103 | "nix-test-runner": "nix-test-runner_3", 104 | "nixpkgs": "nixpkgs_6", 105 | "pre-commit-hooks": "pre-commit-hooks_3" 106 | }, 107 | "locked": { 108 | "lastModified": 1739473963, 109 | "narHash": "sha256-ItAhpjNUzEWd/cgZVyW/jvoGbCec4TK29e1Mnmn1oJE=", 110 | "owner": "nix-community", 111 | "repo": "crate2nix", 112 | "rev": "be31feae9a82c225c0fd1bdf978565dc452a483a", 113 | "type": "github" 114 | }, 115 | "original": { 116 | "owner": "nix-community", 117 | "repo": "crate2nix", 118 | "type": "github" 119 | } 120 | }, 121 | "crate2nix_stable": { 122 | "inputs": { 123 | "cachix": "cachix_2", 124 | "crate2nix_stable": "crate2nix_stable_2", 125 | "devshell": "devshell_2", 126 | "flake-compat": "flake-compat_2", 127 | "flake-parts": "flake-parts_2", 128 | "nix-test-runner": "nix-test-runner_2", 129 | "nixpkgs": "nixpkgs_5", 130 | "pre-commit-hooks": "pre-commit-hooks_2" 131 | }, 132 | "locked": { 133 | "lastModified": 1719760004, 134 | "narHash": "sha256-esWhRnt7FhiYq0CcIxw9pvH+ybOQmWBfHYMtleaMhBE=", 135 | "owner": "nix-community", 136 | "repo": "crate2nix", 137 | "rev": "1dee214bb20855fa3e1e7bb98d28922ddaff8c57", 138 | "type": "github" 139 | }, 140 | "original": { 141 | "owner": "nix-community", 142 | "ref": "0.14.1", 143 | "repo": "crate2nix", 144 | "type": "github" 145 | } 146 | }, 147 | "crate2nix_stable_2": { 148 | "inputs": { 149 | "cachix": "cachix_3", 150 | "crate2nix_stable": "crate2nix_stable_3", 151 | "devshell": "devshell", 152 | "flake-compat": "flake-compat", 153 | "flake-parts": "flake-parts", 154 | "nix-test-runner": "nix-test-runner", 155 | "nixpkgs": "nixpkgs_4", 156 | "pre-commit-hooks": "pre-commit-hooks" 157 | }, 158 | "locked": { 159 | "lastModified": 1712821484, 160 | "narHash": "sha256-rGT3CW64cJS9nlnWPFWSc1iEa3dNZecVVuPVGzcsHe8=", 161 | "owner": "nix-community", 162 | "repo": "crate2nix", 163 | "rev": "42883afcad3823fa5811e967fb7bff54bc3c9d6d", 164 | "type": "github" 165 | }, 166 | "original": { 167 | "owner": "nix-community", 168 | "ref": "0.14.0", 169 | "repo": "crate2nix", 170 | "type": "github" 171 | } 172 | }, 173 | "crate2nix_stable_3": { 174 | "inputs": { 175 | "flake-utils": "flake-utils" 176 | }, 177 | "locked": { 178 | "lastModified": 1702842982, 179 | "narHash": "sha256-A9AowkHIjsy1a4LuiPiVP88FMxyCWK41flZEZOUuwQM=", 180 | "owner": "nix-community", 181 | "repo": "crate2nix", 182 | "rev": "75ac2973affa6b9b4f661a7b592cba6e4f51d426", 183 | "type": "github" 184 | }, 185 | "original": { 186 | "owner": "nix-community", 187 | "ref": "0.12.0", 188 | "repo": "crate2nix", 189 | "type": "github" 190 | } 191 | }, 192 | "devshell": { 193 | "inputs": { 194 | "flake-utils": "flake-utils_2", 195 | "nixpkgs": [ 196 | "crate2nix", 197 | "crate2nix_stable", 198 | "crate2nix_stable", 199 | "nixpkgs" 200 | ] 201 | }, 202 | "locked": { 203 | "lastModified": 1717408969, 204 | "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=", 205 | "owner": "numtide", 206 | "repo": "devshell", 207 | "rev": "1ebbe68d57457c8cae98145410b164b5477761f4", 208 | "type": "github" 209 | }, 210 | "original": { 211 | "owner": "numtide", 212 | "repo": "devshell", 213 | "type": "github" 214 | } 215 | }, 216 | "devshell_2": { 217 | "inputs": { 218 | "flake-utils": "flake-utils_3", 219 | "nixpkgs": [ 220 | "crate2nix", 221 | "crate2nix_stable", 222 | "nixpkgs" 223 | ] 224 | }, 225 | "locked": { 226 | "lastModified": 1717408969, 227 | "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=", 228 | "owner": "numtide", 229 | "repo": "devshell", 230 | "rev": "1ebbe68d57457c8cae98145410b164b5477761f4", 231 | "type": "github" 232 | }, 233 | "original": { 234 | "owner": "numtide", 235 | "repo": "devshell", 236 | "type": "github" 237 | } 238 | }, 239 | "devshell_3": { 240 | "inputs": { 241 | "flake-utils": "flake-utils_4", 242 | "nixpkgs": [ 243 | "crate2nix", 244 | "nixpkgs" 245 | ] 246 | }, 247 | "locked": { 248 | "lastModified": 1711099426, 249 | "narHash": "sha256-HzpgM/wc3aqpnHJJ2oDqPBkNsqWbW0WfWUO8lKu8nGk=", 250 | "owner": "numtide", 251 | "repo": "devshell", 252 | "rev": "2d45b54ca4a183f2fdcf4b19c895b64fbf620ee8", 253 | "type": "github" 254 | }, 255 | "original": { 256 | "owner": "numtide", 257 | "repo": "devshell", 258 | "type": "github" 259 | } 260 | }, 261 | "flake-compat": { 262 | "locked": { 263 | "lastModified": 1696426674, 264 | "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", 265 | "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", 266 | "revCount": 57, 267 | "type": "tarball", 268 | "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz" 269 | }, 270 | "original": { 271 | "type": "tarball", 272 | "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" 273 | } 274 | }, 275 | "flake-compat_2": { 276 | "locked": { 277 | "lastModified": 1696426674, 278 | "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", 279 | "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", 280 | "revCount": 57, 281 | "type": "tarball", 282 | "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz" 283 | }, 284 | "original": { 285 | "type": "tarball", 286 | "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" 287 | } 288 | }, 289 | "flake-compat_3": { 290 | "locked": { 291 | "lastModified": 1696426674, 292 | "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", 293 | "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", 294 | "revCount": 57, 295 | "type": "tarball", 296 | "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz" 297 | }, 298 | "original": { 299 | "type": "tarball", 300 | "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" 301 | } 302 | }, 303 | "flake-parts": { 304 | "inputs": { 305 | "nixpkgs-lib": [ 306 | "crate2nix", 307 | "crate2nix_stable", 308 | "crate2nix_stable", 309 | "nixpkgs" 310 | ] 311 | }, 312 | "locked": { 313 | "lastModified": 1719745305, 314 | "narHash": "sha256-xwgjVUpqSviudEkpQnioeez1Uo2wzrsMaJKJClh+Bls=", 315 | "owner": "hercules-ci", 316 | "repo": "flake-parts", 317 | "rev": "c3c5ecc05edc7dafba779c6c1a61cd08ac6583e9", 318 | "type": "github" 319 | }, 320 | "original": { 321 | "owner": "hercules-ci", 322 | "repo": "flake-parts", 323 | "type": "github" 324 | } 325 | }, 326 | "flake-parts_2": { 327 | "inputs": { 328 | "nixpkgs-lib": [ 329 | "crate2nix", 330 | "crate2nix_stable", 331 | "nixpkgs" 332 | ] 333 | }, 334 | "locked": { 335 | "lastModified": 1719745305, 336 | "narHash": "sha256-xwgjVUpqSviudEkpQnioeez1Uo2wzrsMaJKJClh+Bls=", 337 | "owner": "hercules-ci", 338 | "repo": "flake-parts", 339 | "rev": "c3c5ecc05edc7dafba779c6c1a61cd08ac6583e9", 340 | "type": "github" 341 | }, 342 | "original": { 343 | "owner": "hercules-ci", 344 | "repo": "flake-parts", 345 | "type": "github" 346 | } 347 | }, 348 | "flake-parts_3": { 349 | "inputs": { 350 | "nixpkgs-lib": [ 351 | "crate2nix", 352 | "nixpkgs" 353 | ] 354 | }, 355 | "locked": { 356 | "lastModified": 1712014858, 357 | "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", 358 | "owner": "hercules-ci", 359 | "repo": "flake-parts", 360 | "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", 361 | "type": "github" 362 | }, 363 | "original": { 364 | "owner": "hercules-ci", 365 | "repo": "flake-parts", 366 | "type": "github" 367 | } 368 | }, 369 | "flake-utils": { 370 | "inputs": { 371 | "systems": "systems" 372 | }, 373 | "locked": { 374 | "lastModified": 1694529238, 375 | "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", 376 | "owner": "numtide", 377 | "repo": "flake-utils", 378 | "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", 379 | "type": "github" 380 | }, 381 | "original": { 382 | "owner": "numtide", 383 | "repo": "flake-utils", 384 | "type": "github" 385 | } 386 | }, 387 | "flake-utils_2": { 388 | "inputs": { 389 | "systems": "systems_2" 390 | }, 391 | "locked": { 392 | "lastModified": 1701680307, 393 | "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", 394 | "owner": "numtide", 395 | "repo": "flake-utils", 396 | "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", 397 | "type": "github" 398 | }, 399 | "original": { 400 | "owner": "numtide", 401 | "repo": "flake-utils", 402 | "type": "github" 403 | } 404 | }, 405 | "flake-utils_3": { 406 | "inputs": { 407 | "systems": "systems_3" 408 | }, 409 | "locked": { 410 | "lastModified": 1701680307, 411 | "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", 412 | "owner": "numtide", 413 | "repo": "flake-utils", 414 | "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", 415 | "type": "github" 416 | }, 417 | "original": { 418 | "owner": "numtide", 419 | "repo": "flake-utils", 420 | "type": "github" 421 | } 422 | }, 423 | "flake-utils_4": { 424 | "inputs": { 425 | "systems": "systems_4" 426 | }, 427 | "locked": { 428 | "lastModified": 1701680307, 429 | "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", 430 | "owner": "numtide", 431 | "repo": "flake-utils", 432 | "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", 433 | "type": "github" 434 | }, 435 | "original": { 436 | "owner": "numtide", 437 | "repo": "flake-utils", 438 | "type": "github" 439 | } 440 | }, 441 | "flake-utils_5": { 442 | "inputs": { 443 | "systems": "systems_5" 444 | }, 445 | "locked": { 446 | "lastModified": 1710146030, 447 | "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", 448 | "owner": "numtide", 449 | "repo": "flake-utils", 450 | "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", 451 | "type": "github" 452 | }, 453 | "original": { 454 | "owner": "numtide", 455 | "repo": "flake-utils", 456 | "type": "github" 457 | } 458 | }, 459 | "flake-utils_6": { 460 | "inputs": { 461 | "systems": "systems_6" 462 | }, 463 | "locked": { 464 | "lastModified": 1731533236, 465 | "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", 466 | "owner": "numtide", 467 | "repo": "flake-utils", 468 | "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", 469 | "type": "github" 470 | }, 471 | "original": { 472 | "owner": "numtide", 473 | "repo": "flake-utils", 474 | "type": "github" 475 | } 476 | }, 477 | "gitignore": { 478 | "inputs": { 479 | "nixpkgs": [ 480 | "crate2nix", 481 | "crate2nix_stable", 482 | "crate2nix_stable", 483 | "pre-commit-hooks", 484 | "nixpkgs" 485 | ] 486 | }, 487 | "locked": { 488 | "lastModified": 1709087332, 489 | "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", 490 | "owner": "hercules-ci", 491 | "repo": "gitignore.nix", 492 | "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", 493 | "type": "github" 494 | }, 495 | "original": { 496 | "owner": "hercules-ci", 497 | "repo": "gitignore.nix", 498 | "type": "github" 499 | } 500 | }, 501 | "gitignore_2": { 502 | "inputs": { 503 | "nixpkgs": [ 504 | "crate2nix", 505 | "crate2nix_stable", 506 | "pre-commit-hooks", 507 | "nixpkgs" 508 | ] 509 | }, 510 | "locked": { 511 | "lastModified": 1709087332, 512 | "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", 513 | "owner": "hercules-ci", 514 | "repo": "gitignore.nix", 515 | "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", 516 | "type": "github" 517 | }, 518 | "original": { 519 | "owner": "hercules-ci", 520 | "repo": "gitignore.nix", 521 | "type": "github" 522 | } 523 | }, 524 | "gitignore_3": { 525 | "inputs": { 526 | "nixpkgs": [ 527 | "crate2nix", 528 | "pre-commit-hooks", 529 | "nixpkgs" 530 | ] 531 | }, 532 | "locked": { 533 | "lastModified": 1709087332, 534 | "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", 535 | "owner": "hercules-ci", 536 | "repo": "gitignore.nix", 537 | "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", 538 | "type": "github" 539 | }, 540 | "original": { 541 | "owner": "hercules-ci", 542 | "repo": "gitignore.nix", 543 | "type": "github" 544 | } 545 | }, 546 | "nix-test-runner": { 547 | "flake": false, 548 | "locked": { 549 | "lastModified": 1588761593, 550 | "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=", 551 | "owner": "stoeffel", 552 | "repo": "nix-test-runner", 553 | "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2", 554 | "type": "github" 555 | }, 556 | "original": { 557 | "owner": "stoeffel", 558 | "repo": "nix-test-runner", 559 | "type": "github" 560 | } 561 | }, 562 | "nix-test-runner_2": { 563 | "flake": false, 564 | "locked": { 565 | "lastModified": 1588761593, 566 | "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=", 567 | "owner": "stoeffel", 568 | "repo": "nix-test-runner", 569 | "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2", 570 | "type": "github" 571 | }, 572 | "original": { 573 | "owner": "stoeffel", 574 | "repo": "nix-test-runner", 575 | "type": "github" 576 | } 577 | }, 578 | "nix-test-runner_3": { 579 | "flake": false, 580 | "locked": { 581 | "lastModified": 1588761593, 582 | "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=", 583 | "owner": "stoeffel", 584 | "repo": "nix-test-runner", 585 | "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2", 586 | "type": "github" 587 | }, 588 | "original": { 589 | "owner": "stoeffel", 590 | "repo": "nix-test-runner", 591 | "type": "github" 592 | } 593 | }, 594 | "nixpkgs": { 595 | "locked": { 596 | "lastModified": 1700612854, 597 | "narHash": "sha256-yrQ8osMD+vDLGFX7pcwsY/Qr5PUd6OmDMYJZzZi0+zc=", 598 | "owner": "NixOS", 599 | "repo": "nixpkgs", 600 | "rev": "19cbff58383a4ae384dea4d1d0c823d72b49d614", 601 | "type": "github" 602 | }, 603 | "original": { 604 | "owner": "NixOS", 605 | "ref": "nixos-unstable", 606 | "repo": "nixpkgs", 607 | "type": "github" 608 | } 609 | }, 610 | "nixpkgs_2": { 611 | "locked": { 612 | "lastModified": 1715534503, 613 | "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=", 614 | "owner": "NixOS", 615 | "repo": "nixpkgs", 616 | "rev": "2057814051972fa1453ddfb0d98badbea9b83c06", 617 | "type": "github" 618 | }, 619 | "original": { 620 | "owner": "NixOS", 621 | "ref": "nixos-unstable", 622 | "repo": "nixpkgs", 623 | "type": "github" 624 | } 625 | }, 626 | "nixpkgs_3": { 627 | "locked": { 628 | "lastModified": 1715534503, 629 | "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=", 630 | "owner": "NixOS", 631 | "repo": "nixpkgs", 632 | "rev": "2057814051972fa1453ddfb0d98badbea9b83c06", 633 | "type": "github" 634 | }, 635 | "original": { 636 | "owner": "NixOS", 637 | "ref": "nixos-unstable", 638 | "repo": "nixpkgs", 639 | "type": "github" 640 | } 641 | }, 642 | "nixpkgs_4": { 643 | "locked": { 644 | "lastModified": 1719506693, 645 | "narHash": "sha256-C8e9S7RzshSdHB7L+v9I51af1gDM5unhJ2xO1ywxNH8=", 646 | "path": "/nix/store/4p0avw1s3vf27hspgqsrqs37gxk4i83i-source", 647 | "rev": "b2852eb9365c6de48ffb0dc2c9562591f652242a", 648 | "type": "path" 649 | }, 650 | "original": { 651 | "id": "nixpkgs", 652 | "type": "indirect" 653 | } 654 | }, 655 | "nixpkgs_5": { 656 | "locked": { 657 | "lastModified": 1719506693, 658 | "narHash": "sha256-C8e9S7RzshSdHB7L+v9I51af1gDM5unhJ2xO1ywxNH8=", 659 | "path": "/nix/store/4p0avw1s3vf27hspgqsrqs37gxk4i83i-source", 660 | "rev": "b2852eb9365c6de48ffb0dc2c9562591f652242a", 661 | "type": "path" 662 | }, 663 | "original": { 664 | "id": "nixpkgs", 665 | "type": "indirect" 666 | } 667 | }, 668 | "nixpkgs_6": { 669 | "locked": { 670 | "lastModified": 1712026416, 671 | "narHash": "sha256-N/3VR/9e1NlN49p7kCiATiEY6Tzdo+CbrAG8kqCQKcI=", 672 | "owner": "NixOS", 673 | "repo": "nixpkgs", 674 | "rev": "080a4a27f206d07724b88da096e27ef63401a504", 675 | "type": "github" 676 | }, 677 | "original": { 678 | "id": "nixpkgs", 679 | "type": "indirect" 680 | } 681 | }, 682 | "nixpkgs_7": { 683 | "locked": { 684 | "lastModified": 1744440957, 685 | "narHash": "sha256-FHlSkNqFmPxPJvy+6fNLaNeWnF1lZSgqVCl/eWaJRc4=", 686 | "owner": "NixOS", 687 | "repo": "nixpkgs", 688 | "rev": "26d499fc9f1d567283d5d56fcf367edd815dba1d", 689 | "type": "github" 690 | }, 691 | "original": { 692 | "owner": "NixOS", 693 | "ref": "nixos-24.11", 694 | "repo": "nixpkgs", 695 | "type": "github" 696 | } 697 | }, 698 | "pre-commit-hooks": { 699 | "inputs": { 700 | "flake-compat": [ 701 | "crate2nix", 702 | "crate2nix_stable", 703 | "crate2nix_stable", 704 | "flake-compat" 705 | ], 706 | "gitignore": "gitignore", 707 | "nixpkgs": [ 708 | "crate2nix", 709 | "crate2nix_stable", 710 | "crate2nix_stable", 711 | "nixpkgs" 712 | ], 713 | "nixpkgs-stable": [ 714 | "crate2nix", 715 | "crate2nix_stable", 716 | "crate2nix_stable", 717 | "nixpkgs" 718 | ] 719 | }, 720 | "locked": { 721 | "lastModified": 1719259945, 722 | "narHash": "sha256-F1h+XIsGKT9TkGO3omxDLEb/9jOOsI6NnzsXFsZhry4=", 723 | "owner": "cachix", 724 | "repo": "pre-commit-hooks.nix", 725 | "rev": "0ff4381bbb8f7a52ca4a851660fc7a437a4c6e07", 726 | "type": "github" 727 | }, 728 | "original": { 729 | "owner": "cachix", 730 | "repo": "pre-commit-hooks.nix", 731 | "type": "github" 732 | } 733 | }, 734 | "pre-commit-hooks_2": { 735 | "inputs": { 736 | "flake-compat": [ 737 | "crate2nix", 738 | "crate2nix_stable", 739 | "flake-compat" 740 | ], 741 | "gitignore": "gitignore_2", 742 | "nixpkgs": [ 743 | "crate2nix", 744 | "crate2nix_stable", 745 | "nixpkgs" 746 | ], 747 | "nixpkgs-stable": [ 748 | "crate2nix", 749 | "crate2nix_stable", 750 | "nixpkgs" 751 | ] 752 | }, 753 | "locked": { 754 | "lastModified": 1719259945, 755 | "narHash": "sha256-F1h+XIsGKT9TkGO3omxDLEb/9jOOsI6NnzsXFsZhry4=", 756 | "owner": "cachix", 757 | "repo": "pre-commit-hooks.nix", 758 | "rev": "0ff4381bbb8f7a52ca4a851660fc7a437a4c6e07", 759 | "type": "github" 760 | }, 761 | "original": { 762 | "owner": "cachix", 763 | "repo": "pre-commit-hooks.nix", 764 | "type": "github" 765 | } 766 | }, 767 | "pre-commit-hooks_3": { 768 | "inputs": { 769 | "flake-compat": [ 770 | "crate2nix", 771 | "flake-compat" 772 | ], 773 | "flake-utils": "flake-utils_5", 774 | "gitignore": "gitignore_3", 775 | "nixpkgs": [ 776 | "crate2nix", 777 | "nixpkgs" 778 | ], 779 | "nixpkgs-stable": [ 780 | "crate2nix", 781 | "nixpkgs" 782 | ] 783 | }, 784 | "locked": { 785 | "lastModified": 1712055707, 786 | "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=", 787 | "owner": "cachix", 788 | "repo": "pre-commit-hooks.nix", 789 | "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a", 790 | "type": "github" 791 | }, 792 | "original": { 793 | "owner": "cachix", 794 | "repo": "pre-commit-hooks.nix", 795 | "type": "github" 796 | } 797 | }, 798 | "root": { 799 | "inputs": { 800 | "crate2nix": "crate2nix", 801 | "flake-utils": "flake-utils_6", 802 | "nixpkgs": "nixpkgs_7", 803 | "rust-overlay": "rust-overlay" 804 | } 805 | }, 806 | "rust-overlay": { 807 | "inputs": { 808 | "nixpkgs": [ 809 | "nixpkgs" 810 | ] 811 | }, 812 | "locked": { 813 | "lastModified": 1745029910, 814 | "narHash": "sha256-9CtbfTTQWMoOkXejxc5D+K3z/39wkQQt2YfYJW50tnI=", 815 | "owner": "oxalica", 816 | "repo": "rust-overlay", 817 | "rev": "50fefac8cdfd1587ac6d8678f6181e7d348201d2", 818 | "type": "github" 819 | }, 820 | "original": { 821 | "owner": "oxalica", 822 | "repo": "rust-overlay", 823 | "type": "github" 824 | } 825 | }, 826 | "systems": { 827 | "locked": { 828 | "lastModified": 1681028828, 829 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 830 | "owner": "nix-systems", 831 | "repo": "default", 832 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 833 | "type": "github" 834 | }, 835 | "original": { 836 | "owner": "nix-systems", 837 | "repo": "default", 838 | "type": "github" 839 | } 840 | }, 841 | "systems_2": { 842 | "locked": { 843 | "lastModified": 1681028828, 844 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 845 | "owner": "nix-systems", 846 | "repo": "default", 847 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 848 | "type": "github" 849 | }, 850 | "original": { 851 | "owner": "nix-systems", 852 | "repo": "default", 853 | "type": "github" 854 | } 855 | }, 856 | "systems_3": { 857 | "locked": { 858 | "lastModified": 1681028828, 859 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 860 | "owner": "nix-systems", 861 | "repo": "default", 862 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 863 | "type": "github" 864 | }, 865 | "original": { 866 | "owner": "nix-systems", 867 | "repo": "default", 868 | "type": "github" 869 | } 870 | }, 871 | "systems_4": { 872 | "locked": { 873 | "lastModified": 1681028828, 874 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 875 | "owner": "nix-systems", 876 | "repo": "default", 877 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 878 | "type": "github" 879 | }, 880 | "original": { 881 | "owner": "nix-systems", 882 | "repo": "default", 883 | "type": "github" 884 | } 885 | }, 886 | "systems_5": { 887 | "locked": { 888 | "lastModified": 1681028828, 889 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 890 | "owner": "nix-systems", 891 | "repo": "default", 892 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 893 | "type": "github" 894 | }, 895 | "original": { 896 | "owner": "nix-systems", 897 | "repo": "default", 898 | "type": "github" 899 | } 900 | }, 901 | "systems_6": { 902 | "locked": { 903 | "lastModified": 1681028828, 904 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 905 | "owner": "nix-systems", 906 | "repo": "default", 907 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 908 | "type": "github" 909 | }, 910 | "original": { 911 | "owner": "nix-systems", 912 | "repo": "default", 913 | "type": "github" 914 | } 915 | } 916 | }, 917 | "root": "root", 918 | "version": 7 919 | } 920 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | inputs = { 3 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; 4 | flake-utils.url = "github:numtide/flake-utils"; 5 | crate2nix.url = "github:nix-community/crate2nix"; 6 | 7 | rust-overlay = { 8 | url = "github:oxalica/rust-overlay"; 9 | inputs.nixpkgs.follows = "nixpkgs"; 10 | }; 11 | }; 12 | 13 | outputs = 14 | inputs@{ 15 | flake-utils, 16 | nixpkgs, 17 | crate2nix, 18 | rust-overlay, 19 | ... 20 | }: 21 | (flake-utils.lib.eachDefaultSystem ( 22 | system: 23 | let 24 | pkgs = import nixpkgs { 25 | inherit system; 26 | overlays = [ 27 | crate2nix.overlays.default 28 | rust-overlay.overlays.default 29 | ]; 30 | }; 31 | 32 | applyPatch = 33 | { src, patches }: 34 | pkgs.stdenvNoCC.mkDerivation { 35 | name = "${src.name}-patched"; 36 | inherit src patches; 37 | patchFlags = [ "--strip 1" ]; 38 | 39 | nativeBuildInputs = [ pkgs.patch ]; 40 | 41 | installPhase = '' 42 | mkdir -p $out 43 | cp -r . $out/ 44 | ''; 45 | }; 46 | 47 | duckdbCrate = applyPatch { 48 | src = pkgs.fetchCrate { 49 | pname = "duckdb"; 50 | version = "1.0.0"; 51 | sha256 = "sha256-XC1+mjocHl0GUSNNNi3pC+BXQEdG6IzeizzAnR5lzMA="; 52 | }; 53 | patches = [ patches/duckdb+1.0.0.patch ]; 54 | }; 55 | 56 | duckdbLoadableMacrosCrate = applyPatch { 57 | src = pkgs.fetchCrate { 58 | pname = "duckdb-loadable-macros"; 59 | version = "0.1.2"; 60 | sha256 = "sha256-sZzChlJ8O/S/qULlXfeV6UuavbMShIQbiUqPFYb1XKw="; 61 | }; 62 | patches = [ patches/duckdb-loadable-macros+0.1.2.patch ]; 63 | }; 64 | 65 | libduckdbSysCrate = applyPatch { 66 | src = pkgs.fetchCrate { 67 | pname = "libduckdb-sys"; 68 | version = "1.0.0"; 69 | sha256 = "sha256-k9v0RVHOGZoNzyHGu+IKNAfPr6iTDeNPu7VF8kGMRgw="; 70 | }; 71 | patches = [ patches/libduckdb-sys+1.0.0.patch ]; 72 | }; 73 | 74 | vendoredSrc = pkgs.stdenvNoCC.mkDerivation { 75 | name = "duckdb-protobuf-src-with-vendor"; 76 | src = ./.; 77 | 78 | buildPhase = '' 79 | mkdir -p packages/vendor/duckdb 80 | mkdir -p packages/vendor/duckdb-loadable-macros 81 | mkdir -p packages/vendor/libduckdb-sys 82 | 83 | cp -r ${duckdbCrate}/* packages/vendor/duckdb/ 84 | cp -r ${duckdbLoadableMacrosCrate}/* packages/vendor/duckdb-loadable-macros/ 85 | cp -r ${libduckdbSysCrate}/* packages/vendor/libduckdb-sys/ 86 | ''; 87 | 88 | installPhase = '' 89 | mkdir -p $out 90 | cp -r . $out/ 91 | ''; 92 | }; 93 | 94 | buildRustCrateForPkgs = 95 | crate: 96 | pkgs.buildRustCrate.override { 97 | rustc = pkgs.rust-bin.stable.latest.default; 98 | cargo = pkgs.rust-bin.stable.latest.default; 99 | }; 100 | 101 | generatedCargoNix = inputs.crate2nix.tools.${system}.generatedCargoNix { 102 | name = "duckdb_protobuf"; 103 | src = vendoredSrc; 104 | }; 105 | 106 | cargoNix = import generatedCargoNix { 107 | inherit pkgs buildRustCrateForPkgs; 108 | }; 109 | 110 | duckdb_protobuf = cargoNix.workspaceMembers.duckdb_protobuf.build; 111 | duckdb_metadata_bin = cargoNix.workspaceMembers.duckdb_metadata_bin.build; 112 | 113 | extensionVersion = "v0.0.1"; 114 | apiVersion = "v0.0.1"; 115 | 116 | platform = 117 | if system == "x86_64-linux" then 118 | "linux_amd64" 119 | else if system == "aarch64-linux" then 120 | "linux_arm64" 121 | else if system == "x86_64-darwin" then 122 | "osx_amd64" 123 | else if system == "aarch64-darwin" then 124 | "osx_arm64" 125 | else 126 | throw "Unsupported platform: ${system}"; 127 | in 128 | rec { 129 | devShells.default = pkgs.mkShell { 130 | packages = [ 131 | pkgs.crate2nix 132 | ]; 133 | }; 134 | packages = { 135 | default = pkgs.stdenv.mkDerivation { 136 | name = "duckdb-protobuf-extension"; 137 | 138 | phases = [ 139 | "buildPhase" 140 | "installPhase" 141 | ]; 142 | 143 | buildPhase = '' 144 | LIBRARY_PATH=$(find ${duckdb_protobuf.lib} -type f -name "*.dylib" -o -name "*.so" -o -name "*.dll" | head -n 1) 145 | 146 | ${duckdb_metadata_bin}/bin/duckdb_metadata \ 147 | --input "$LIBRARY_PATH" \ 148 | --output protobuf.duckdb_extension \ 149 | --extension-version ${extensionVersion} \ 150 | --duckdb-api-version ${apiVersion} \ 151 | --platform ${platform} \ 152 | --extension-abi-type C_STRUCT 153 | ''; 154 | 155 | installPhase = '' 156 | mkdir -p $out/${platform} 157 | cp protobuf.duckdb_extension $out/${platform} 158 | ''; 159 | }; 160 | }; 161 | } 162 | )); 163 | } 164 | -------------------------------------------------------------------------------- /packages/duckdb_metadata/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "duckdb_metadata" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | 8 | [dependencies] 9 | gimli = "0.31.0" 10 | thiserror = "1.0.63" 11 | bytemuck = { version = "1.16.1", features = ["derive"] } 12 | -------------------------------------------------------------------------------- /packages/duckdb_metadata/src/lib.rs: -------------------------------------------------------------------------------- 1 | use bytemuck::{Pod, Zeroable}; 2 | use gimli::leb128; 3 | use std::io; 4 | use thiserror::Error; 5 | 6 | #[repr(C)] 7 | #[derive(Clone, Copy, Pod, Zeroable)] 8 | pub struct MetadataFields { 9 | pub meta_8: [u8; 32], 10 | pub meta_7: [u8; 32], 11 | pub meta_6: [u8; 32], 12 | pub extension_abi_type: [u8; 32], 13 | pub extension_version: [u8; 32], 14 | pub duckdb_api_version: [u8; 32], 15 | pub platform: [u8; 32], 16 | pub metadata_version: [u8; 32], 17 | pub signature: [u8; 256], 18 | } 19 | 20 | impl MetadataFields { 21 | pub fn write(&self, mut write: impl io::Write) -> io::Result<()> { 22 | let metadata_fields_size = std::mem::size_of::(); 23 | 24 | let mut metadata_field_section_bytes = [0u8; 10]; 25 | let metadata_fields_section_len = leb128::write::unsigned( 26 | &mut { &mut metadata_field_section_bytes[..] }, 27 | metadata_fields_size as u64, 28 | ) 29 | .unwrap(); 30 | let metadata_fields_section_len_bytes = 31 | &metadata_field_section_bytes[0..metadata_fields_section_len]; 32 | 33 | let custom_section_name = b"duckdb_signature"; 34 | 35 | let mut custom_section_name_bytes = [0u8; 10]; 36 | let custom_section_name_len = leb128::write::unsigned( 37 | &mut { &mut custom_section_name_bytes[..] }, 38 | custom_section_name.len() as u64, 39 | ) 40 | .unwrap(); 41 | let custom_section_name_len_bytes = &custom_section_name_bytes[..custom_section_name_len]; 42 | 43 | let metadata_fields_size = custom_section_name_len_bytes.len() 44 | + custom_section_name.len() 45 | + metadata_fields_section_len_bytes.len() 46 | + metadata_fields_size; 47 | 48 | write.write_all(&[0])?; 49 | 50 | { 51 | let mut bytes = [0u8; 10]; 52 | let len = leb128::write::unsigned(&mut { &mut bytes[..] }, metadata_fields_size as u64) 53 | .unwrap(); 54 | let length_bytes = &bytes[0..len]; 55 | 56 | write.write_all(length_bytes)?; 57 | } 58 | 59 | write.write_all(custom_section_name_len_bytes)?; 60 | write.write_all(custom_section_name)?; 61 | 62 | write.write_all(metadata_fields_section_len_bytes)?; 63 | 64 | let metadata_bytes = bytemuck::bytes_of(self); 65 | write.write_all(metadata_bytes)?; 66 | 67 | Ok(()) 68 | } 69 | } 70 | 71 | #[derive(Error, Debug)] 72 | #[error("input too long")] 73 | pub struct InputTooLongError; 74 | 75 | pub fn pad_32(bytes: &[u8]) -> Result<[u8; 32], InputTooLongError> { 76 | let mut result = [0u8; 32]; 77 | if bytes.len() > result.len() { 78 | return Err(InputTooLongError); 79 | } 80 | 81 | let copy_len = bytes.len(); 82 | result[..copy_len].copy_from_slice(&bytes[..copy_len]); 83 | Ok(result) 84 | } 85 | 86 | #[cfg(test)] 87 | mod tests { 88 | use super::*; 89 | use std::io::Cursor; 90 | 91 | #[test] 92 | fn test_create_metadata_section() -> Result<(), InputTooLongError> { 93 | // tail of http://welsch.lu/duckdb/prql/v0.0.19/v1.0.0/osx_arm64/prql.duckdb_extension.gz 94 | let expected: [u8; 0x216] = [ 95 | 0x00, 0x93, 0x04, 0x10, 0x64, 0x75, 0x63, 0x6B, 0x64, 0x62, 0x5F, 0x73, 0x69, 0x67, 96 | 0x6E, 0x61, 0x74, 0x75, 0x72, 0x65, 0x80, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 97 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 98 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 99 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 100 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 101 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 102 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 103 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 104 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 105 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x30, 0x2E, 0x30, 106 | 0x2E, 0x31, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 107 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 108 | 0x76, 0x31, 0x2E, 0x30, 0x2E, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 109 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 110 | 0x00, 0x00, 0x00, 0x00, 0x6F, 0x73, 0x78, 0x5F, 0x61, 0x72, 0x6D, 0x36, 0x34, 0x00, 111 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 112 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 113 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 114 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 115 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 116 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 117 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 118 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 119 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 120 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 121 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 122 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 123 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 124 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 125 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 126 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 127 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 128 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 129 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 130 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 131 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 132 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 133 | 0x00, 0x00, 134 | ]; 135 | 136 | let args = MetadataFields { 137 | meta_8: [0; 32], 138 | meta_7: [0; 32], 139 | meta_6: [0; 32], 140 | extension_abi_type: [0; 32], 141 | extension_version: pad_32(b"v0.0.19")?, 142 | duckdb_api_version: pad_32(b"v1.0.0")?, 143 | platform: pad_32(b"osx_arm64")?, 144 | metadata_version: pad_32(b"4")?, 145 | signature: [0; 256], 146 | }; 147 | 148 | let mut buffer = Cursor::new(Vec::new()); 149 | args.write(&mut buffer).unwrap(); 150 | 151 | let result = buffer.into_inner(); 152 | 153 | assert_eq!(result, &expected); 154 | 155 | Ok(()) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /packages/duckdb_metadata_bin/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "duckdb_metadata_bin" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [[bin]] 7 | name = "duckdb_metadata" 8 | path = "src/main.rs" 9 | 10 | [dependencies] 11 | clap = { version = "4.5.9", features = ["derive"] } 12 | duckdb_metadata = { path = "../duckdb_metadata" } 13 | anyhow = "1.0.86" 14 | -------------------------------------------------------------------------------- /packages/duckdb_metadata_bin/src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{Context, Result}; 2 | use clap::Parser; 3 | use std::fs::File; 4 | use std::io; 5 | use std::path::PathBuf; 6 | 7 | use duckdb_metadata::{pad_32, MetadataFields}; 8 | 9 | #[derive(Parser, Debug)] 10 | #[clap(author, version, about, long_about = None)] 11 | struct Args { 12 | #[clap(long)] 13 | output: PathBuf, 14 | 15 | #[clap(long)] 16 | input: PathBuf, 17 | 18 | #[clap(long)] 19 | extension_version: String, 20 | 21 | #[clap(long)] 22 | duckdb_api_version: String, 23 | 24 | /// Full list on https://duckdb.org/docs/extensions/working_with_extensions.html#platforms 25 | #[clap(long)] 26 | platform: String, 27 | 28 | #[clap(long, default_value = "4")] 29 | metadata_version: String, 30 | 31 | #[clap(long, default_value = "CPP")] 32 | extension_abi_type: String, 33 | } 34 | 35 | fn main() -> Result<()> { 36 | let args = Args::parse(); 37 | 38 | let extension_version = 39 | pad_32(args.extension_version.as_bytes()).context("extension_version")?; 40 | let duckdb_api_version = 41 | pad_32(args.duckdb_api_version.as_bytes()).context("duckdb_api_version")?; 42 | let platform = pad_32(args.platform.as_bytes()).context("platform")?; 43 | let metadata_version = pad_32(args.metadata_version.as_bytes()).context("metadata_version")?; 44 | let extension_abi_type = 45 | pad_32(args.extension_abi_type.as_bytes()).context("extension_abi_type")?; 46 | 47 | let metadata_fields = MetadataFields { 48 | meta_8: [0; 32], 49 | meta_7: [0; 32], 50 | meta_6: [0; 32], 51 | extension_abi_type, 52 | extension_version, 53 | duckdb_api_version, 54 | platform, 55 | metadata_version, 56 | signature: [0; 256], 57 | }; 58 | 59 | let mut input_file = File::open(&args.input) 60 | .with_context(|| format!("failed to open input file: {:?}", args.input))?; 61 | 62 | let mut output_file = File::create(&args.output) 63 | .with_context(|| format!("failed to create output file: {:?}", args.output))?; 64 | 65 | io::copy(&mut input_file, &mut output_file)?; 66 | 67 | metadata_fields 68 | .write(&mut output_file) 69 | .context("failed to write metadata to output file")?; 70 | 71 | println!("output generated {:?}", args.output); 72 | 73 | Ok(()) 74 | } 75 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "duckdb_protobuf" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | crate-type = ["cdylib"] 8 | 9 | [dependencies] 10 | anyhow = "1.0.86" 11 | duckdb = { version = "1.0.0", features = ["vtab-loadable", "loadable_extension"] } 12 | duckdb-loadable-macros = "0.1.2" 13 | prost = "0.13.0" 14 | prost-reflect = "0.14.0" 15 | protobuf = "3.5.0" 16 | glob = "0.3.1" 17 | byteorder = "1.5.0" 18 | log = "0.4.21" 19 | ouroboros = "0.18.4" 20 | strum = { version = "0.26.3", features = ["derive"] } 21 | crossbeam = "0.8.4" 22 | 23 | [dev-dependencies] 24 | anyhow = "1.0" 25 | prost = "0.13.1" 26 | prost-build = "0.13.1" 27 | 28 | [[test]] 29 | name = "it" 30 | path = "tests/it/main.rs" 31 | 32 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/src/filtered_dynamic_message.rs: -------------------------------------------------------------------------------- 1 | use prost::bytes::{Buf, BufMut}; 2 | use prost::encoding::{DecodeContext, WireType}; 3 | use prost::{DecodeError, Message}; 4 | use prost_reflect::{DynamicMessage, UnknownField}; 5 | use std::collections::HashSet; 6 | 7 | #[derive(Debug, Clone)] 8 | pub struct FilteredDynamicMessage { 9 | message: DynamicMessage, 10 | accepted_fields: HashSet, 11 | } 12 | 13 | impl FilteredDynamicMessage { 14 | pub fn new(message: DynamicMessage, accepted_fields: HashSet) -> FilteredDynamicMessage { 15 | FilteredDynamicMessage { 16 | message, 17 | accepted_fields, 18 | } 19 | } 20 | 21 | pub fn into(self) -> DynamicMessage { 22 | self.message 23 | } 24 | } 25 | 26 | impl Message for FilteredDynamicMessage { 27 | fn encode_raw(&self, buf: &mut impl BufMut) 28 | where 29 | Self: Sized, 30 | { 31 | self.message.encode_raw(buf) 32 | } 33 | 34 | fn merge_field( 35 | &mut self, 36 | number: u32, 37 | wire_type: WireType, 38 | buf: &mut impl Buf, 39 | ctx: DecodeContext, 40 | ) -> Result<(), DecodeError> 41 | where 42 | Self: Sized, 43 | { 44 | if !self.accepted_fields.contains(&number) { 45 | let _field = UnknownField::decode_value(number, wire_type, buf, ctx)?; 46 | return Ok(()); 47 | } 48 | 49 | self.message.merge_field(number, wire_type, buf, ctx) 50 | } 51 | 52 | fn encoded_len(&self) -> usize { 53 | self.message.encoded_len() 54 | } 55 | 56 | fn clear(&mut self) { 57 | self.message.clear() 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/src/io.rs: -------------------------------------------------------------------------------- 1 | use anyhow::format_err; 2 | use byteorder::{BigEndian, ReadBytesExt}; 3 | use ouroboros::self_referencing; 4 | use protobuf::CodedInputStream; 5 | use std::error::Error; 6 | use std::fs::File; 7 | use std::io; 8 | use std::path::{Path, PathBuf}; 9 | use strum::{AsRefStr, EnumIter, EnumString, IntoEnumIterator}; 10 | 11 | #[derive(Copy, Clone, EnumString, EnumIter, AsRefStr)] 12 | pub enum LengthKind { 13 | BigEndianFixed, 14 | Varint, 15 | SingleMessagePerFile, 16 | } 17 | 18 | pub fn parse + IntoEnumIterator + AsRef>( 19 | value: &str, 20 | ) -> Result { 21 | Ok(T::from_str(value).map_err(|err| { 22 | format_err!( 23 | "{}: expected one of: {}, got: {}", 24 | err, 25 | T::iter() 26 | .map(|it| format!("{}", it.as_ref())) 27 | .collect::>() 28 | .join(", "), 29 | value 30 | ) 31 | })?) 32 | } 33 | 34 | #[derive(Copy, Clone)] 35 | pub enum DelimitedLengthKind { 36 | BigEndianFixed, 37 | Varint, 38 | } 39 | 40 | #[self_referencing] 41 | pub struct LengthDelimitedRecordsReader { 42 | length_kind: DelimitedLengthKind, 43 | path: PathBuf, 44 | inner: File, 45 | 46 | #[borrows(mut inner)] 47 | #[not_covariant] 48 | reader: CodedInputStream<'this>, 49 | } 50 | 51 | pub struct Record { 52 | pub bytes: Vec, 53 | pub position: u64, 54 | pub size: u32, 55 | } 56 | 57 | impl LengthDelimitedRecordsReader { 58 | pub fn create(inner: File, length_kind: DelimitedLengthKind, path: PathBuf) -> Self { 59 | LengthDelimitedRecordsReaderBuilder { 60 | length_kind, 61 | path, 62 | inner, 63 | reader_builder: |it| CodedInputStream::new(it), 64 | } 65 | .build() 66 | } 67 | 68 | fn get_next(&mut self) -> Result { 69 | let length_kind = *self.borrow_length_kind(); 70 | Ok(self.with_reader_mut(move |reader| { 71 | let position = reader.pos(); 72 | let len = match length_kind { 73 | DelimitedLengthKind::BigEndianFixed => reader.read_u32::()?, 74 | DelimitedLengthKind::Varint => reader.read_raw_varint32()?, 75 | }; 76 | 77 | let mut buf = vec![0; len as usize]; 78 | ::read_exact(reader, &mut buf)?; 79 | 80 | Ok::<_, io::Error>(Record { 81 | bytes: buf, 82 | position, 83 | size: len, 84 | }) 85 | })?) 86 | } 87 | 88 | pub fn try_get_next(&mut self) -> Result, io::Error> { 89 | match self.get_next() { 90 | Ok(it) => Ok(Some(it)), 91 | Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => Ok(None), 92 | Err(err) => Err(err.into()), 93 | } 94 | } 95 | 96 | pub fn path(&self) -> &Path { 97 | self.borrow_path().as_path() 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod filtered_dynamic_message; 2 | mod io; 3 | mod read; 4 | mod types; 5 | mod vtab; 6 | 7 | use std::error::Error; 8 | 9 | use crate::vtab::ProtobufVTab; 10 | use duckdb::ffi; 11 | use duckdb::Connection; 12 | use duckdb_loadable_macros::duckdb_entrypoint_c_api; 13 | 14 | #[duckdb_entrypoint_c_api(ext_name = "protobuf", min_duckdb_version = "v0.0.1")] 15 | fn protobuf_init(conn: Connection) -> Result<(), Box> { 16 | conn.register_table_function_local_init::("protobuf")?; 17 | 18 | Ok(()) 19 | } 20 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/src/read.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::ffi::CString; 3 | use std::marker::PhantomData; 4 | use std::slice; 5 | 6 | use anyhow::{bail, format_err}; 7 | use duckdb::vtab::{DataChunk, LogicalType, LogicalTypeId}; 8 | use prost_reflect::{Cardinality, DynamicMessage, FieldDescriptor, Kind, ReflectMessage, Value}; 9 | 10 | pub fn write_to_output( 11 | mappings: &[u64], 12 | columns_state: &mut HashMap, 13 | value: &DynamicMessage, 14 | output: &DataChunk, 15 | max_rows: usize, 16 | row_idx: usize, 17 | ) -> Result<(), anyhow::Error> { 18 | let column_key = &ColumnKey::empty(); 19 | let fields = value.descriptor().fields().collect::>(); 20 | for (output_field_idx, field_idx) in mappings.iter().enumerate() { 21 | let field_idx = *field_idx as usize; 22 | if field_idx >= fields.len() { 23 | continue; 24 | } 25 | 26 | let field_descriptor = &fields[field_idx]; 27 | let column_vector = output.get_vector(output_field_idx); 28 | let value = value.get_field(&field_descriptor); 29 | 30 | let column_key = column_key.field(&field_descriptor); 31 | 32 | write_column( 33 | columns_state, 34 | &column_key, 35 | &value, 36 | &field_descriptor, 37 | column_vector, 38 | max_rows, 39 | row_idx, 40 | )?; 41 | } 42 | 43 | Ok(()) 44 | } 45 | 46 | pub fn write_message( 47 | columns_state: &mut HashMap, 48 | column_key: &ColumnKey, 49 | value: &DynamicMessage, 50 | output: &impl VectorAccessor, 51 | max_rows: usize, 52 | row_idx: usize, 53 | ) -> Result<(), anyhow::Error> { 54 | for (field_idx, field_descriptor) in value.descriptor().fields().enumerate() { 55 | let column_vector = output.get_vector(field_idx); 56 | let value = value.get_field(&field_descriptor); 57 | 58 | let column_key = column_key.field(&field_descriptor); 59 | 60 | write_column( 61 | columns_state, 62 | &column_key, 63 | &value, 64 | &field_descriptor, 65 | column_vector, 66 | max_rows, 67 | row_idx, 68 | )?; 69 | } 70 | 71 | Ok(()) 72 | } 73 | 74 | pub struct MyFlatVector { 75 | _phantom_data: PhantomData, 76 | ptr: duckdb::ffi::duckdb_vector, 77 | capacity: usize, 78 | } 79 | 80 | impl MyFlatVector { 81 | pub unsafe fn with_capacity(ptr: duckdb::ffi::duckdb_vector, capacity: usize) -> Self { 82 | Self { 83 | _phantom_data: Default::default(), 84 | ptr, 85 | capacity, 86 | } 87 | } 88 | 89 | fn as_mut_ptr(&self) -> *mut T { 90 | unsafe { duckdb::ffi::duckdb_vector_get_data(self.ptr).cast() } 91 | } 92 | 93 | pub fn as_mut_slice(&mut self) -> &mut [T] { 94 | unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.capacity) } 95 | } 96 | } 97 | 98 | pub fn write_column( 99 | columns_state: &mut HashMap, 100 | column_key: &ColumnKey, 101 | value: &Value, 102 | field_descriptor: &FieldDescriptor, 103 | column: duckdb::ffi::duckdb_vector, 104 | max_rows: usize, 105 | row_idx: usize, 106 | ) -> Result<(), anyhow::Error> { 107 | match field_descriptor.cardinality() { 108 | Cardinality::Repeated => { 109 | let column_key = column_key.extending(ColumnKeyElement::List); 110 | 111 | let mut list_entries_vector = unsafe { 112 | MyFlatVector::::with_capacity(column, max_rows) 113 | }; 114 | let list_entry = &mut list_entries_vector.as_mut_slice()[row_idx]; 115 | 116 | let values = value 117 | .as_list() 118 | .ok_or_else(|| format_err!("expected list"))?; 119 | 120 | let next_offset_ref = columns_state.get_mut(&column_key); 121 | let next_offset = if let Some(it) = &next_offset_ref { 122 | **it 123 | } else { 124 | 0 125 | }; 126 | 127 | let len_u64 = u64::try_from(values.len())?; 128 | 129 | list_entry.offset = next_offset; 130 | list_entry.length = len_u64; 131 | 132 | let new_next_offset = next_offset + len_u64; 133 | 134 | if let Some(it) = next_offset_ref { 135 | *it = new_next_offset; 136 | } else { 137 | columns_state.insert(column_key.clone(), new_next_offset); 138 | } 139 | 140 | let new_length = new_next_offset; 141 | 142 | unsafe { duckdb::ffi::duckdb_list_vector_reserve(column, new_length) }; 143 | unsafe { duckdb::ffi::duckdb_list_vector_set_size(column, new_length) }; 144 | 145 | let child_vector = unsafe { duckdb::ffi::duckdb_list_vector_get_child(column) }; 146 | 147 | for (idx, value) in values.iter().enumerate() { 148 | let row_idx = next_offset as usize + idx; 149 | 150 | write_single_column( 151 | columns_state, 152 | &column_key, 153 | value, 154 | field_descriptor, 155 | child_vector, 156 | new_length as usize, 157 | row_idx, 158 | )?; 159 | } 160 | } 161 | Cardinality::Optional | Cardinality::Required => { 162 | write_single_column( 163 | columns_state, 164 | column_key, 165 | value, 166 | field_descriptor, 167 | column, 168 | max_rows, 169 | row_idx, 170 | )?; 171 | } 172 | } 173 | 174 | Ok(()) 175 | } 176 | 177 | pub fn write_single_column( 178 | columns_state: &mut HashMap, 179 | column_key: &ColumnKey, 180 | value: &Value, 181 | field_descriptor: &FieldDescriptor, 182 | column: duckdb::ffi::duckdb_vector, 183 | max_rows: usize, 184 | row_idx: usize, 185 | ) -> Result<(), anyhow::Error> { 186 | match field_descriptor.kind() { 187 | Kind::Message(message_descriptor) 188 | if message_descriptor.full_name() == "google.protobuf.Timestamp" => 189 | { 190 | let message = value 191 | .as_message() 192 | .ok_or_else(|| format_err!("expected message"))?; 193 | let seconds = 194 | message 195 | .get_field(&message_descriptor.get_field(1).ok_or_else(|| { 196 | format_err!("expected field 1 for google.protobuf.Timestamp") 197 | })?) 198 | .as_i64() 199 | .ok_or_else(|| format_err!("expected i64"))?; 200 | 201 | let nanos = 202 | message 203 | .get_field(&message_descriptor.get_field(2).ok_or_else(|| { 204 | format_err!("expected field 2 for google.protobuf.Timestamp") 205 | })?) 206 | .as_i32() 207 | .ok_or_else(|| format_err!("expected i32"))?; 208 | 209 | let mut vector = unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 210 | vector.as_mut_slice()[row_idx] = seconds * 1000000 + (nanos as i64 / 1000); 211 | } 212 | Kind::Message(..) => { 213 | let message = value 214 | .as_message() 215 | .ok_or_else(|| format_err!("expected message"))?; 216 | 217 | let source = unsafe { StructVector::new(column) }; 218 | 219 | write_message( 220 | columns_state, 221 | column_key, 222 | message, 223 | &source, 224 | max_rows, 225 | row_idx, 226 | )?; 227 | } 228 | Kind::Enum(enum_descriptor) => { 229 | let enum_value = value 230 | .as_enum_number() 231 | .ok_or_else(|| format_err!("expected enum value"))?; 232 | 233 | let enum_value_descriptor = enum_descriptor 234 | .get_value(enum_value) 235 | .unwrap_or_else(|| enum_descriptor.default_value()); 236 | 237 | let (idx, _) = enum_descriptor 238 | .values() 239 | .enumerate() 240 | .find(|(_, it)| it.number() == enum_value_descriptor.number()) 241 | .unwrap(); 242 | 243 | let column_type = 244 | unsafe { duckdb::ffi::duckdb_vector_get_column_type(column) }; 245 | 246 | let logical_type = LogicalTypeId::from(unsafe { duckdb::ffi::duckdb_enum_internal_type(column_type) }); 247 | 248 | match logical_type { 249 | LogicalTypeId::UTinyint => { 250 | let mut vector = unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 251 | vector.as_mut_slice()[row_idx] = idx as _; 252 | } 253 | LogicalTypeId::USmallint => { 254 | let mut vector = 255 | unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 256 | vector.as_mut_slice()[row_idx] = idx as _; 257 | } 258 | LogicalTypeId::UInteger => { 259 | let mut vector = 260 | unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 261 | vector.as_mut_slice()[row_idx] = idx as _; 262 | } 263 | _ => bail!("unknown enum column type {:?}", logical_type), 264 | } 265 | } 266 | Kind::String => { 267 | let value = value 268 | .as_str() 269 | .ok_or_else(|| format_err!("expected string"))?; 270 | let value = CString::new(value)?; 271 | 272 | unsafe { 273 | duckdb::ffi::duckdb_vector_assign_string_element( 274 | column, 275 | row_idx as u64, 276 | value.as_ptr(), 277 | ) 278 | }; 279 | } 280 | Kind::Double => { 281 | let value = value 282 | .as_f64() 283 | .ok_or_else(|| format_err!("expected double"))?; 284 | let mut vector = unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 285 | vector.as_mut_slice()[row_idx] = value; 286 | } 287 | Kind::Float => { 288 | let value = value 289 | .as_f32() 290 | .ok_or_else(|| format_err!("expected float"))?; 291 | let mut vector = unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 292 | vector.as_mut_slice()[row_idx] = value; 293 | } 294 | Kind::Int32 => { 295 | let value = value 296 | .as_i32() 297 | .ok_or_else(|| format_err!("expected int32"))?; 298 | let mut vector = unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 299 | vector.as_mut_slice()[row_idx] = value; 300 | } 301 | Kind::Int64 => { 302 | let value = value 303 | .as_i64() 304 | .ok_or_else(|| format_err!("expected int64"))?; 305 | let mut vector = unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 306 | vector.as_mut_slice()[row_idx] = value; 307 | } 308 | Kind::Uint32 => { 309 | let value = value 310 | .as_u32() 311 | .ok_or_else(|| format_err!("expected uint32"))?; 312 | let mut vector = unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 313 | vector.as_mut_slice()[row_idx] = value; 314 | } 315 | Kind::Uint64 => { 316 | let value = value 317 | .as_u64() 318 | .ok_or_else(|| format_err!("expected uint64"))?; 319 | let mut vector = unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 320 | vector.as_mut_slice()[row_idx] = value; 321 | } 322 | Kind::Bool => { 323 | let value = value 324 | .as_bool() 325 | .ok_or_else(|| format_err!("expected bool"))?; 326 | let mut vector = unsafe { MyFlatVector::::with_capacity(column, max_rows) }; 327 | vector.as_mut_slice()[row_idx] = value; 328 | } 329 | _ => { 330 | bail!("unhandled field type"); 331 | } 332 | }; 333 | 334 | Ok(()) 335 | } 336 | 337 | #[derive(Hash, Eq, PartialEq, Clone)] 338 | pub enum ColumnKeyElement { 339 | Field { field_tag: u32 }, 340 | List, 341 | } 342 | 343 | #[derive(Hash, Eq, PartialEq, Clone)] 344 | pub struct ColumnKey { 345 | pub elements: Vec, 346 | } 347 | 348 | impl ColumnKey { 349 | pub fn field(&self, field: &FieldDescriptor) -> ColumnKey { 350 | self.extending(ColumnKeyElement::Field { 351 | field_tag: field.number(), 352 | }) 353 | } 354 | 355 | pub fn extending(&self, key: ColumnKeyElement) -> ColumnKey { 356 | let mut elements = self.elements.clone(); 357 | elements.push(key); 358 | 359 | ColumnKey { elements } 360 | } 361 | pub fn empty() -> ColumnKey { 362 | ColumnKey { elements: vec![] } 363 | } 364 | } 365 | 366 | pub trait VectorAccessor { 367 | fn get_vector(&self, column_idx: usize) -> duckdb::ffi::duckdb_vector; 368 | } 369 | 370 | impl VectorAccessor for DataChunk { 371 | fn get_vector(&self, column_idx: usize) -> duckdb::ffi::duckdb_vector { 372 | let chunk = self.get_ptr(); 373 | 374 | unsafe { duckdb::ffi::duckdb_data_chunk_get_vector(chunk, column_idx as u64) } 375 | } 376 | } 377 | 378 | struct StructVector(duckdb::ffi::duckdb_vector); 379 | 380 | impl StructVector { 381 | unsafe fn new(value: duckdb::ffi::duckdb_vector) -> Self { 382 | Self(value) 383 | } 384 | } 385 | 386 | impl VectorAccessor for StructVector { 387 | fn get_vector(&self, idx: usize) -> duckdb::ffi::duckdb_vector { 388 | unsafe { duckdb::ffi::duckdb_struct_vector_get_child(self.0, idx as u64) } 389 | } 390 | } 391 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/src/types.rs: -------------------------------------------------------------------------------- 1 | use anyhow::format_err; 2 | use duckdb::vtab::{LogicalType, LogicalTypeId}; 3 | use prost_reflect::{Cardinality, FieldDescriptor, Kind}; 4 | 5 | pub fn into_logical_type(field: &FieldDescriptor) -> Result { 6 | Ok(match field.cardinality() { 7 | Cardinality::Optional | Cardinality::Required => into_logical_type_single(field)?, 8 | Cardinality::Repeated => LogicalType::list(&into_logical_type_single(field)?), 9 | }) 10 | } 11 | 12 | fn into_logical_type_single(field: &FieldDescriptor) -> Result { 13 | let value = match field.kind() { 14 | Kind::Message(message_descriptor) 15 | if message_descriptor.full_name() == "google.protobuf.Timestamp" => 16 | { 17 | LogicalType::new(LogicalTypeId::Timestamp) 18 | } 19 | Kind::Message(message_descriptor) => { 20 | let fields = message_descriptor 21 | .fields() 22 | .collect::>(); 23 | 24 | let fields = fields 25 | .iter() 26 | .map(|field| Ok((field.name(), into_logical_type(&field)?))) 27 | .collect::, anyhow::Error>>()?; 28 | 29 | LogicalType::struct_type(fields.as_slice()) 30 | } 31 | Kind::Enum(descriptor) => { 32 | let names = descriptor.values().collect::>(); 33 | let names = names.iter().map(|it| it.name()).collect::>(); 34 | LogicalType::enumeration(names.as_slice()) 35 | } 36 | Kind::Double => LogicalType::new(LogicalTypeId::Double), 37 | Kind::Float => LogicalType::new(LogicalTypeId::Float), 38 | Kind::Int32 => LogicalType::new(LogicalTypeId::Integer), 39 | Kind::Int64 => LogicalType::new(LogicalTypeId::Bigint), 40 | Kind::Uint32 => LogicalType::new(LogicalTypeId::UInteger), 41 | Kind::Uint64 => LogicalType::new(LogicalTypeId::UBigint), 42 | Kind::Bool => LogicalType::new(LogicalTypeId::Boolean), 43 | Kind::String => LogicalType::new(LogicalTypeId::Varchar), 44 | logical_type => { 45 | return Err(format_err!( 46 | "unhandled field: {}, type: {:?}", 47 | field.name(), 48 | logical_type, 49 | ) 50 | .into()) 51 | } 52 | }; 53 | 54 | Ok(value) 55 | } 56 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/src/vtab.rs: -------------------------------------------------------------------------------- 1 | use crate::filtered_dynamic_message::FilteredDynamicMessage; 2 | use crate::io::{parse, DelimitedLengthKind, LengthDelimitedRecordsReader, LengthKind, Record}; 3 | use crate::read::{write_to_output, MyFlatVector, VectorAccessor}; 4 | use crate::types::into_logical_type; 5 | use anyhow::{format_err, Context}; 6 | use crossbeam::queue::ArrayQueue; 7 | use duckdb::vtab::{ 8 | BindInfo, DataChunk, Free, FunctionInfo, InitInfo, LogicalType, LogicalTypeId, VTab, 9 | VTabLocalData, 10 | }; 11 | use prost::Message; 12 | use prost_reflect::{DescriptorPool, DynamicMessage, MessageDescriptor, ReflectMessage}; 13 | use std::error::Error; 14 | use std::ffi::CString; 15 | use std::fs::File; 16 | use std::io::Read; 17 | use std::ops::{Deref, DerefMut}; 18 | use std::path::{Path, PathBuf}; 19 | use std::ptr::null_mut; 20 | 21 | pub struct Parameters { 22 | pub files: String, 23 | pub descriptor_bytes: Vec, 24 | pub message_name: String, 25 | pub shared_message_descriptor: MessageDescriptor, 26 | pub length_kind: LengthKind, 27 | pub include_filename: bool, 28 | pub include_position: bool, 29 | pub include_size: bool, 30 | } 31 | 32 | impl Parameters { 33 | pub fn from_bind_info(bind: &BindInfo) -> Result { 34 | let files = bind 35 | .get_named_parameter("files") 36 | .ok_or_else(|| format_err!("missing argument `files`"))? 37 | .to_string(); 38 | 39 | let descriptor = bind 40 | .get_named_parameter("descriptors") 41 | .ok_or_else(|| format_err!("missing parameter `descriptor`"))? 42 | .to_string(); 43 | 44 | let descriptor_bytes = (|| -> Result, anyhow::Error> { 45 | let mut file = File::open(descriptor)?; 46 | let mut buffer = Vec::new(); 47 | file.read_to_end(&mut buffer)?; 48 | 49 | Ok(buffer) 50 | })() 51 | .with_context(|| format_err!("field `descriptors`"))?; 52 | 53 | let shared_descriptor_pool = DescriptorPool::decode(descriptor_bytes.as_slice())?; 54 | 55 | let message_name = bind 56 | .get_named_parameter("message_type") 57 | .ok_or_else(|| format_err!("missing parameter `message_type`"))? 58 | .to_string(); 59 | 60 | let message_descriptor = shared_descriptor_pool 61 | .get_message_by_name(&message_name.as_str()) 62 | .ok_or_else(|| format_err!("message type not found in `descriptor`"))?; 63 | 64 | let length_kind = bind 65 | .get_named_parameter("delimiter") 66 | .ok_or_else(|| format_err!("missing parameter `delimiter`"))?; 67 | 68 | let length_kind = parse::(&length_kind.to_string()) 69 | .map_err(|err| format_err!("when parsing parameter delimiter: {}", err))?; 70 | 71 | let include_filename = bind 72 | .get_named_parameter("filename") 73 | .map(|value| value.to_int64() != 0) 74 | .unwrap_or(false); 75 | 76 | let include_position = bind 77 | .get_named_parameter("position") 78 | .map(|value| value.to_int64() != 0) 79 | .unwrap_or(false); 80 | 81 | let include_size = bind 82 | .get_named_parameter("size") 83 | .map(|value| value.to_int64() != 0) 84 | .unwrap_or(false); 85 | 86 | Ok(Self { 87 | files, 88 | descriptor_bytes, 89 | message_name, 90 | shared_message_descriptor: message_descriptor, 91 | length_kind, 92 | include_filename, 93 | include_position, 94 | include_size, 95 | }) 96 | } 97 | 98 | pub fn message_descriptor(&self) -> Result { 99 | let descriptor_pool = DescriptorPool::decode(self.descriptor_bytes.as_slice())?; 100 | 101 | let message_descriptor = descriptor_pool 102 | .get_message_by_name(&self.message_name) 103 | .unwrap(); 104 | 105 | Ok(message_descriptor) 106 | } 107 | 108 | pub fn values() -> Vec<(String, LogicalType)> { 109 | vec![ 110 | ( 111 | "files".to_string(), 112 | LogicalType::new(LogicalTypeId::Varchar), 113 | ), 114 | ( 115 | "message_type".to_string(), 116 | LogicalType::new(LogicalTypeId::Varchar), 117 | ), 118 | ( 119 | "descriptors".to_string(), 120 | LogicalType::new(LogicalTypeId::Varchar), 121 | ), 122 | ( 123 | "delimiter".to_string(), 124 | LogicalType::new(LogicalTypeId::Varchar), 125 | ), 126 | ( 127 | "filename".to_string(), 128 | LogicalType::new(LogicalTypeId::Boolean), 129 | ), 130 | ( 131 | "position".to_string(), 132 | LogicalType::new(LogicalTypeId::Boolean), 133 | ), 134 | ("size".to_string(), LogicalType::new(LogicalTypeId::Boolean)), 135 | ] 136 | } 137 | } 138 | pub struct GlobalState { 139 | queue: ArrayQueue, 140 | column_indices: Vec, 141 | } 142 | 143 | impl GlobalState { 144 | pub fn new( 145 | params: &Parameters, 146 | column_indices: Vec, 147 | ) -> Result { 148 | let tasks = { 149 | let mut tasks = vec![]; 150 | let items = glob::glob(params.files.as_str())?; 151 | for item in items { 152 | let item = item?; 153 | tasks.push(item); 154 | } 155 | 156 | tasks 157 | }; 158 | 159 | if tasks.is_empty() { 160 | return Err(format_err!("no files matching glob found {}", params.files)); 161 | } 162 | 163 | let queue = { 164 | let queue = ArrayQueue::new(tasks.len()); 165 | 166 | for item in tasks { 167 | queue.push(item).unwrap(); 168 | } 169 | 170 | queue 171 | }; 172 | 173 | Ok(GlobalState { 174 | queue, 175 | column_indices, 176 | }) 177 | } 178 | } 179 | 180 | pub struct ProtobufVTab; 181 | 182 | impl VTab for ProtobufVTab { 183 | type InitData = Handle; 184 | type BindData = Handle; 185 | 186 | unsafe fn bind( 187 | bind: &BindInfo, 188 | data: *mut Self::BindData, 189 | ) -> duckdb::Result<(), Box> { 190 | Ok(Self::bind(bind, data).map_err(format_error_with_causes)?) 191 | } 192 | 193 | unsafe fn init( 194 | init_info: &InitInfo, 195 | data: *mut Self::InitData, 196 | ) -> duckdb::Result<(), Box> { 197 | Ok(Self::init(init_info, data).map_err(format_error_with_causes)?) 198 | } 199 | 200 | unsafe fn func( 201 | func: &FunctionInfo, 202 | output: &mut DataChunk, 203 | ) -> duckdb::Result<(), Box> { 204 | Ok(Self::func(func, output).map_err(format_error_with_causes)?) 205 | } 206 | 207 | fn named_parameters() -> Option> { 208 | Some(Parameters::values()) 209 | } 210 | 211 | fn supports_pushdown() -> bool { 212 | true 213 | } 214 | } 215 | 216 | impl ProtobufVTab { 217 | fn bind(bind: &BindInfo, data: *mut ::BindData) -> Result<(), anyhow::Error> { 218 | let data = unsafe { &mut *data }; 219 | data.init(); 220 | 221 | let params = Parameters::from_bind_info(bind)?; 222 | 223 | for field_descriptor in params.shared_message_descriptor.fields() { 224 | bind.add_result_column( 225 | field_descriptor.name().as_ref(), 226 | into_logical_type(&field_descriptor)?, 227 | ); 228 | } 229 | 230 | if params.include_filename { 231 | bind.add_result_column("filename", LogicalType::new(LogicalTypeId::Varchar)); 232 | } 233 | 234 | if params.include_position { 235 | bind.add_result_column("position", LogicalType::new(LogicalTypeId::UBigint)); 236 | } 237 | 238 | if params.include_size { 239 | bind.add_result_column("size", LogicalType::new(LogicalTypeId::UBigint)); 240 | } 241 | 242 | data.assign(params); 243 | 244 | Ok(()) 245 | } 246 | 247 | fn init( 248 | init_info: &InitInfo, 249 | data: *mut ::InitData, 250 | ) -> Result<(), anyhow::Error> { 251 | let data = unsafe { &mut *data }; 252 | data.init(); 253 | 254 | let bind_data = unsafe { &*init_info.get_bind_data::<::BindData>() }; 255 | let column_indices = init_info.get_column_indices(); 256 | 257 | let new_global_state = GlobalState::new(bind_data, column_indices)?; 258 | init_info.set_max_threads(new_global_state.queue.len() as _); 259 | data.assign(new_global_state); 260 | 261 | Ok(()) 262 | } 263 | 264 | fn func(func: &FunctionInfo, output: &mut DataChunk) -> duckdb::Result<(), anyhow::Error> { 265 | let bind_data = unsafe { &mut *func.get_bind_data::<::BindData>() }; 266 | let init_data = unsafe { &mut *func.get_init_data::<::InitData>() }; 267 | let local_init_data = 268 | unsafe { &mut *func.get_local_init_data::<::LocalInitData>() }; 269 | 270 | let parameters: &Parameters = bind_data.deref(); 271 | 272 | let local_descriptor = local_init_data.local_descriptor.clone(); 273 | 274 | let mut state_container = StateContainer { 275 | local_state: local_init_data, 276 | global_state: init_data, 277 | parameters, 278 | }; 279 | 280 | let available_chunk_size = output.flat_vector(0).capacity(); 281 | let mut items = 0; 282 | 283 | let mut column_information = Default::default(); 284 | 285 | let message = { 286 | let message = DynamicMessage::new(local_descriptor.clone()); 287 | let fields: Vec<_> = local_descriptor.fields().collect(); 288 | 289 | let message = FilteredDynamicMessage::new( 290 | message, 291 | init_data 292 | .column_indices 293 | .iter() 294 | .filter_map(|it| { 295 | let it = *it as usize; 296 | if it >= fields.len() { 297 | return None; 298 | } 299 | 300 | Some(fields[it].number()) 301 | }) 302 | .collect(), 303 | ); 304 | 305 | message 306 | }; 307 | 308 | for output_row_idx in 0..available_chunk_size { 309 | let StateContainerValue { 310 | path_reference, 311 | size, 312 | bytes, 313 | position, 314 | } = match state_container.next_message()? { 315 | None => break, 316 | Some(message_info) => message_info, 317 | }; 318 | 319 | let mut message = message.clone(); 320 | message.merge(bytes.as_slice())?; 321 | let message = message.into(); 322 | 323 | write_to_output( 324 | &init_data.column_indices, 325 | &mut column_information, 326 | &message, 327 | output, 328 | available_chunk_size, 329 | output_row_idx, 330 | )?; 331 | 332 | let mut field_offset = message.descriptor().fields().len(); 333 | 334 | if parameters.include_filename { 335 | if let Some((field_offset, _)) = init_data 336 | .column_indices 337 | .iter() 338 | .enumerate() 339 | .find(|(_, it)| (**it as usize) == (field_offset)) 340 | { 341 | let it = (|| -> Option { 342 | let value = CString::new(path_reference.path().to_str()?).ok()?; 343 | Some(value) 344 | })(); 345 | 346 | let column = output.get_vector(field_offset); 347 | 348 | match it { 349 | None => unsafe { 350 | let validity = duckdb::ffi::duckdb_vector_get_validity(column); 351 | duckdb::ffi::duckdb_validity_set_row_invalid( 352 | validity, 353 | output_row_idx as _, 354 | ); 355 | }, 356 | Some(value) => unsafe { 357 | duckdb::ffi::duckdb_vector_assign_string_element( 358 | column, 359 | output_row_idx as _, 360 | value.as_ptr(), 361 | ) 362 | }, 363 | } 364 | } 365 | 366 | field_offset += 1; 367 | } 368 | 369 | if parameters.include_position { 370 | if let Some((field_offset, _)) = init_data 371 | .column_indices 372 | .iter() 373 | .enumerate() 374 | .find(|(_, it)| (**it as usize) == (field_offset)) 375 | { 376 | let column = output.get_vector(field_offset); 377 | let mut vector = 378 | unsafe { MyFlatVector::::with_capacity(column, available_chunk_size) }; 379 | vector.as_mut_slice()[output_row_idx] = position as _; 380 | } 381 | 382 | field_offset += 1; 383 | } 384 | 385 | if parameters.include_size { 386 | if let Some((field_offset, _)) = init_data 387 | .column_indices 388 | .iter() 389 | .enumerate() 390 | .find(|(_, it)| (**it as usize) == (field_offset)) 391 | { 392 | let column = output.get_vector(field_offset); 393 | let mut vector = 394 | unsafe { MyFlatVector::::with_capacity(column, available_chunk_size) }; 395 | vector.as_mut_slice()[output_row_idx] = size as _; 396 | } 397 | 398 | field_offset += 1; 399 | } 400 | 401 | items += 1; 402 | } 403 | 404 | output.set_len(items); 405 | 406 | Ok(()) 407 | } 408 | } 409 | 410 | struct StateContainer<'a> { 411 | local_state: &'a mut LocalState, 412 | global_state: &'a GlobalState, 413 | parameters: &'a Parameters, 414 | } 415 | 416 | enum PathReference<'a> { 417 | Borrowed(&'a Path), 418 | Owned(PathBuf), 419 | } 420 | 421 | impl<'a> PathReference<'a> { 422 | pub fn path(&self) -> &Path { 423 | match self { 424 | PathReference::Borrowed(it) => *it, 425 | PathReference::Owned(it) => it.as_path(), 426 | } 427 | } 428 | } 429 | 430 | struct StateContainerValue<'a> { 431 | path_reference: PathReference<'a>, 432 | bytes: Vec, 433 | size: usize, 434 | position: u64, 435 | } 436 | 437 | impl StateContainer<'_> { 438 | fn next_message(&mut self) -> Result, anyhow::Error> { 439 | let mut value = match self.local_state.current.take() { 440 | Some(it) => it, 441 | None => { 442 | let Some(next_file_path) = self.global_state.queue.pop() else { 443 | return Ok(None); 444 | }; 445 | 446 | let mut next_file = File::open(&next_file_path)?; 447 | match self.parameters.length_kind { 448 | LengthKind::BigEndianFixed => LengthDelimitedRecordsReader::create( 449 | next_file, 450 | DelimitedLengthKind::BigEndianFixed, 451 | next_file_path, 452 | ), 453 | LengthKind::Varint => LengthDelimitedRecordsReader::create( 454 | next_file, 455 | DelimitedLengthKind::Varint, 456 | next_file_path, 457 | ), 458 | LengthKind::SingleMessagePerFile => { 459 | let mut bytes = Vec::new(); 460 | next_file.read_to_end(&mut bytes)?; 461 | let size = bytes.len(); 462 | return Ok(Some(StateContainerValue { 463 | bytes, 464 | path_reference: PathReference::Owned(next_file_path), 465 | position: 0, 466 | size, 467 | })); 468 | } 469 | } 470 | } 471 | }; 472 | 473 | let Some(Record { 474 | position, 475 | size, 476 | bytes: next_message, 477 | }) = value.try_get_next()? 478 | else { 479 | return Ok(None); 480 | }; 481 | 482 | self.local_state.current = Some(value); 483 | Ok(Some(StateContainerValue { 484 | path_reference: PathReference::Borrowed( 485 | self.local_state.current.as_ref().unwrap().path(), 486 | ), 487 | bytes: next_message, 488 | size: size as _, 489 | position, 490 | })) 491 | } 492 | } 493 | 494 | #[repr(C)] 495 | pub struct LocalState { 496 | current: Option, 497 | local_descriptor: MessageDescriptor, 498 | } 499 | 500 | impl VTabLocalData for ProtobufVTab { 501 | type LocalInitData = Handle; 502 | 503 | fn local_init( 504 | init_info: &InitInfo, 505 | data: *mut Self::LocalInitData, 506 | ) -> duckdb::Result<(), Box> { 507 | let bind_data = unsafe { &*init_info.get_bind_data::<::BindData>() }; 508 | let local_descriptor = bind_data.message_descriptor()?; 509 | 510 | let data = unsafe { &mut *data }; 511 | data.init(); 512 | 513 | data.assign(LocalState { 514 | current: None, 515 | local_descriptor, 516 | }); 517 | 518 | Ok(()) 519 | } 520 | } 521 | 522 | #[repr(C)] 523 | pub struct Handle { 524 | inner: *mut T, 525 | } 526 | 527 | impl Handle { 528 | pub fn assign(&mut self, inner: T) { 529 | self.inner = Box::into_raw(Box::new(inner)); 530 | } 531 | pub fn init(&mut self) { 532 | self.inner = null_mut(); 533 | } 534 | } 535 | 536 | impl Deref for Handle { 537 | type Target = T; 538 | 539 | fn deref(&self) -> &Self::Target { 540 | if self.inner.is_null() { 541 | panic!("unable to deref non-null handle") 542 | } 543 | 544 | unsafe { &*self.inner } 545 | } 546 | } 547 | 548 | impl DerefMut for Handle { 549 | fn deref_mut(&mut self) -> &mut Self::Target { 550 | unsafe { &mut *self.inner } 551 | } 552 | } 553 | 554 | impl Free for Handle { 555 | fn free(&mut self) { 556 | unsafe { 557 | if self.inner.is_null() { 558 | return; 559 | } 560 | 561 | drop(Box::from_raw(self.inner)); 562 | } 563 | } 564 | } 565 | 566 | fn format_error_with_causes(error: anyhow::Error) -> anyhow::Error { 567 | format_err!( 568 | "{}", 569 | error 570 | .chain() 571 | .map(|cause| cause.to_string()) 572 | .collect::>() 573 | .join(": ") 574 | ) 575 | } 576 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/tests/.gitignore: -------------------------------------------------------------------------------- 1 | /generated 2 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/tests/it/main.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::Write; 3 | use std::path::Path; 4 | use std::sync::Once; 5 | 6 | use anyhow::Result; 7 | use duckdb::{Config, Connection}; 8 | use prost::Message; 9 | 10 | static INIT: Once = Once::new(); 11 | 12 | fn setup() { 13 | INIT.call_once(|| { 14 | compile_protos().expect("Failed to compile protobufs"); 15 | generate_test_data().expect("Failed to generate test data"); 16 | }); 17 | } 18 | 19 | fn compile_protos() -> Result<(), Box> { 20 | let proto_path = "tests/protos/user.proto"; 21 | let descriptor_dir = "tests/generated"; 22 | let out_dir = "tests/src"; 23 | 24 | std::fs::create_dir_all(descriptor_dir)?; 25 | std::fs::create_dir_all(out_dir)?; 26 | 27 | prost_build::Config::new() 28 | .out_dir(out_dir) 29 | .file_descriptor_set_path("tests/generated/descriptor.pb") 30 | .compile_protos(&[proto_path], &[Path::new("tests/protos")])?; 31 | 32 | Ok(()) 33 | } 34 | 35 | fn generate_test_data() -> Result<(), Box> { 36 | // Include the generated Rust code for the protobuf messages 37 | mod user { 38 | include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/src/user.rs")); 39 | } 40 | 41 | // Create some example User messages 42 | let users = [ 43 | user::User { 44 | name: "Alice".to_string(), 45 | id: 1, 46 | }, 47 | user::User { 48 | name: "Bob".to_string(), 49 | id: 2, 50 | }, 51 | user::User { 52 | name: "Charlie".to_string(), 53 | id: 3, 54 | }, 55 | ]; 56 | 57 | let out_dir = "tests/generated/data"; 58 | std::fs::create_dir_all(out_dir)?; 59 | 60 | // Serialize the messages to binary files 61 | for (i, user) in users.iter().enumerate() { 62 | let mut buf = Vec::new(); 63 | user.encode(&mut buf)?; 64 | 65 | let mut file = File::create(format!("{out_dir}/user_{}.bin", i))?; 66 | file.write_all(&buf)?; 67 | 68 | println!("Generated test data for user {}: {:?}", i, user); 69 | } 70 | 71 | Ok(()) 72 | } 73 | 74 | #[test] 75 | fn test_setup_creates_files() { 76 | setup(); 77 | 78 | for i in 0..3 { 79 | let file_path = format!("tests/generated/data/user_{}.bin", i); 80 | assert!( 81 | Path::new(&file_path).exists(), 82 | "File {} should exist", 83 | file_path 84 | ); 85 | } 86 | } 87 | 88 | #[test] 89 | fn test_query_protobuf_data() -> Result<()> { 90 | setup(); 91 | 92 | let config = Config::default().allow_unsigned_extensions()?; 93 | let conn = Connection::open_in_memory_with_flags(config)?; 94 | 95 | conn.execute("LOAD '../../target/release/protobuf.duckdb_extension'", [])?; 96 | println!("DuckDB extension loaded successfully."); 97 | 98 | let mut stmt = conn.prepare( 99 | " 100 | SELECT * FROM protobuf( 101 | descriptors = './tests/generated/descriptor.pb', 102 | files = './tests/generated/data/**/*.bin', 103 | message_type = 'user.User', 104 | delimiter = 'SingleMessagePerFile' 105 | ) 106 | LIMIT 10; 107 | ", 108 | )?; 109 | 110 | let mut rows = stmt.query([])?; 111 | 112 | let mut results = Vec::new(); 113 | while let Some(row) = rows.next()? { 114 | let name: String = row.get(0)?; 115 | let id: i32 = row.get(1)?; 116 | results.push((name, id)); 117 | } 118 | println!("Query result: {results:?}"); 119 | 120 | assert_eq!(results.len(), 3, "Expected 3 rows"); 121 | assert_eq!(results[0].0, "Alice", "Expected first name to be 'Alice'"); 122 | assert_eq!(results[0].1, 1, "Expected first id to be 1"); 123 | Ok(()) 124 | } 125 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/tests/protos/user.proto: -------------------------------------------------------------------------------- 1 | 2 | syntax = "proto3"; 3 | 4 | package user; 5 | 6 | message User { 7 | string name = 1; 8 | int32 id = 2; 9 | } 10 | 11 | -------------------------------------------------------------------------------- /packages/duckdb_protobuf/tests/src/user.rs: -------------------------------------------------------------------------------- 1 | // This file is @generated by prost-build. 2 | #[allow(clippy::derive_partial_eq_without_eq)] 3 | #[derive(Clone, PartialEq, ::prost::Message)] 4 | pub struct User { 5 | #[prost(string, tag = "1")] 6 | pub name: ::prost::alloc::string::String, 7 | #[prost(int32, tag = "2")] 8 | pub id: i32, 9 | } 10 | -------------------------------------------------------------------------------- /packages/serving/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /packages/serving/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serving", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "license": "MIT", 6 | "scripts": { 7 | "deploy": "wrangler" 8 | }, 9 | "devDependencies": { 10 | "@cloudflare/workers-types": "^4.20241218.0", 11 | "@types/semver": "^7.5.8", 12 | "prettier": "^3.4.2", 13 | "wrangler": "^3.99.0" 14 | }, 15 | "dependencies": { 16 | "semver": "^7.6.3" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /packages/serving/src/worker.ts: -------------------------------------------------------------------------------- 1 | import semver from "semver"; 2 | import { WorkerEntrypoint } from "cloudflare:workers"; 3 | 4 | type Env = { 5 | BUCKET: R2Bucket; 6 | }; 7 | 8 | const versionMappings = [{ range: "^1.0.0", apiVersion: "v0.0.1" }]; 9 | 10 | export default class extends WorkerEntrypoint { 11 | async fetch(request: Request) { 12 | const url = new URL(request.url); 13 | 14 | const pathSegments = url.pathname.split("/"); 15 | pathSegments.shift(); 16 | 17 | const requestedVersion = pathSegments.slice().shift(); 18 | if (!requestedVersion) { 19 | return new Response("not found", { status: 404 }); 20 | } 21 | 22 | const releaseMapping = versionMappings.find((entry) => 23 | semver.satisfies(requestedVersion, entry.range), 24 | ); 25 | if (!releaseMapping) { 26 | return new Response("not found", { status: 404 }); 27 | } 28 | 29 | const apiVersion = releaseMapping.apiVersion; 30 | 31 | const path = ["duckdb-api-version", apiVersion] 32 | .concat(pathSegments.slice(1)) 33 | .join("/"); 34 | 35 | const object = await this.env.BUCKET.get(path); 36 | if (!object) { 37 | return new Response("not found", { status: 404 }); 38 | } 39 | 40 | return new Response(object.body); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /packages/serving/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Visit https://aka.ms/tsconfig to read more about this file */ 4 | 5 | /* Projects */ 6 | // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ 7 | // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ 8 | // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ 9 | // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ 10 | // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ 11 | // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ 12 | 13 | /* Language and Environment */ 14 | "target": "es2021", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ 15 | // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ 16 | // "jsx": "preserve", /* Specify what JSX code is generated. */ 17 | // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ 18 | // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ 19 | // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ 20 | // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ 21 | // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ 22 | // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ 23 | // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ 24 | // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ 25 | // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ 26 | 27 | /* Modules */ 28 | "module": "esnext", /* Specify what module code is generated. */ 29 | // "rootDir": "./", /* Specify the root folder within your source files. */ 30 | "moduleResolution": "nodeNext", /* Specify how TypeScript looks up a file from a given module specifier. */ 31 | // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ 32 | // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ 33 | // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ 34 | // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ 35 | "types": [ 36 | "@cloudflare/workers-types" 37 | ], /* Specify type package names to be included without being referenced in a source file. */ 38 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 39 | // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ 40 | // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ 41 | // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ 42 | // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ 43 | // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ 44 | // "resolveJsonModule": true, /* Enable importing .json files. */ 45 | // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ 46 | // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ 47 | 48 | /* JavaScript Support */ 49 | // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ 50 | // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ 51 | // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ 52 | 53 | /* Emit */ 54 | // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ 55 | // "declarationMap": true, /* Create sourcemaps for d.ts files. */ 56 | // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ 57 | // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ 58 | // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ 59 | // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ 60 | // "outDir": "./", /* Specify an output folder for all emitted files. */ 61 | // "removeComments": true, /* Disable emitting comments. */ 62 | "noEmit": true, /* Disable emitting files from a compilation. */ 63 | // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ 64 | // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ 65 | // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ 66 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 67 | // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ 68 | // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ 69 | // "newLine": "crlf", /* Set the newline character for emitting files. */ 70 | // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ 71 | // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ 72 | // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ 73 | // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ 74 | // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ 75 | 76 | /* Interop Constraints */ 77 | // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ 78 | // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ 79 | // "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */ 80 | // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ 81 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 82 | // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ 83 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 84 | 85 | /* Type Checking */ 86 | "strict": true, /* Enable all strict type-checking options. */ 87 | // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ 88 | // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ 89 | // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ 90 | // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ 91 | // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ 92 | // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ 93 | // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ 94 | // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ 95 | // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ 96 | // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ 97 | // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ 98 | // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ 99 | // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ 100 | // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ 101 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ 102 | // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ 103 | // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ 104 | // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ 105 | 106 | /* Completeness */ 107 | // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ 108 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /packages/serving/wrangler.toml: -------------------------------------------------------------------------------- 1 | #:schema node_modules/wrangler/config-schema.json 2 | name = "duckdb_protobuf_serving" 3 | main = "src/worker.ts" 4 | compatibility_date = "2024-06-20" 5 | compatibility_flags = ["nodejs_compat"] 6 | 7 | [[r2_buckets]] 8 | binding = "BUCKET" 9 | bucket_name = "duckdb-extensions" 10 | 11 | [[routes]] 12 | pattern = "duckdb.0xcaff.xyz" 13 | custom_domain = true 14 | -------------------------------------------------------------------------------- /packages/serving/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@cloudflare/kv-asset-handler@0.3.4": 6 | version "0.3.4" 7 | resolved "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.3.4.tgz" 8 | integrity sha512-YLPHc8yASwjNkmcDMQMY35yiWjoKAKnhUbPRszBRS0YgH+IXtsMp61j+yTcnCE3oO2DgP0U3iejLC8FTtKDC8Q== 9 | dependencies: 10 | mime "^3.0.0" 11 | 12 | "@cloudflare/workerd-darwin-64@1.20241218.0": 13 | version "1.20241218.0" 14 | resolved "https://registry.yarnpkg.com/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20241218.0.tgz#cdcdeb2d0c60c6c5de538370570544b32213c5fe" 15 | integrity sha512-8rveQoxtUvlmORKqTWgjv2ycM8uqWox0u9evn3zd2iWKdou5sncFwH517ZRLI3rq9P31ZLmCQBZ0gloFsTeY6w== 16 | 17 | "@cloudflare/workerd-darwin-arm64@1.20241218.0": 18 | version "1.20241218.0" 19 | resolved "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20241218.0.tgz" 20 | integrity sha512-be59Ad9nmM9lCkhHqmTs/uZ3JVZt8NJ9Z0PY+B0xnc5z6WwmV2lj0RVLtq7xJhQsQJA189zt5rXqDP6J+2mu7Q== 21 | 22 | "@cloudflare/workerd-linux-64@1.20241218.0": 23 | version "1.20241218.0" 24 | resolved "https://registry.yarnpkg.com/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20241218.0.tgz#87302ea1e1b7a04b8486958f659f75ab8a142084" 25 | integrity sha512-MzpSBcfZXRxrYWxQ4pVDYDrUbkQuM62ssl4ZtHH8J35OAeGsWFAYji6MkS2SpVwVcvacPwJXIF4JSzp4xKImKw== 26 | 27 | "@cloudflare/workerd-linux-arm64@1.20241218.0": 28 | version "1.20241218.0" 29 | resolved "https://registry.yarnpkg.com/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20241218.0.tgz#560f20883e1afb684199aab03eaeb45804c03dde" 30 | integrity sha512-RIuJjPxpNqvwIs52vQsXeRMttvhIjgg9NLjjFa3jK8Ijnj8c3ZDru9Wqi48lJP07yDFIRr4uDMMqh/y29YQi2A== 31 | 32 | "@cloudflare/workerd-windows-64@1.20241218.0": 33 | version "1.20241218.0" 34 | resolved "https://registry.yarnpkg.com/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20241218.0.tgz#f62b6cdb7a970375721c22ef70a9c238c4bb5f9b" 35 | integrity sha512-tO1VjlvK3F6Yb2d1jgEy/QBYl//9Pyv3K0j+lq8Eu7qdfm0IgKwSRgDWLept84/qmNsQfausZ4JdNGxTf9xsxQ== 36 | 37 | "@cloudflare/workers-types@^4.20241218.0": 38 | version "4.20241218.0" 39 | resolved "https://registry.yarnpkg.com/@cloudflare/workers-types/-/workers-types-4.20241218.0.tgz#05ba65449306db07082d45ef9ad16a984cc81e5e" 40 | integrity sha512-Y0brjmJHcAZBXOPI7lU5hbiXglQWniA1kQjot2ata+HFimyjPPcz+4QWBRrmWcMPo0OadR2Vmac7WStDLpvz0w== 41 | 42 | "@cspotcode/source-map-support@0.8.1": 43 | version "0.8.1" 44 | resolved "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz" 45 | integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw== 46 | dependencies: 47 | "@jridgewell/trace-mapping" "0.3.9" 48 | 49 | "@esbuild-plugins/node-globals-polyfill@^0.2.3": 50 | version "0.2.3" 51 | resolved "https://registry.npmjs.org/@esbuild-plugins/node-globals-polyfill/-/node-globals-polyfill-0.2.3.tgz" 52 | integrity sha512-r3MIryXDeXDOZh7ih1l/yE9ZLORCd5e8vWg02azWRGj5SPTuoh69A2AIyn0Z31V/kHBfZ4HgWJ+OK3GTTwLmnw== 53 | 54 | "@esbuild-plugins/node-modules-polyfill@^0.2.2": 55 | version "0.2.2" 56 | resolved "https://registry.npmjs.org/@esbuild-plugins/node-modules-polyfill/-/node-modules-polyfill-0.2.2.tgz" 57 | integrity sha512-LXV7QsWJxRuMYvKbiznh+U1ilIop3g2TeKRzUxOG5X3YITc8JyyTa90BmLwqqv0YnX4v32CSlG+vsziZp9dMvA== 58 | dependencies: 59 | escape-string-regexp "^4.0.0" 60 | rollup-plugin-node-polyfills "^0.2.1" 61 | 62 | "@esbuild/android-arm64@0.17.19": 63 | version "0.17.19" 64 | resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.17.19.tgz#bafb75234a5d3d1b690e7c2956a599345e84a2fd" 65 | integrity sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA== 66 | 67 | "@esbuild/android-arm@0.17.19": 68 | version "0.17.19" 69 | resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.17.19.tgz#5898f7832c2298bc7d0ab53701c57beb74d78b4d" 70 | integrity sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A== 71 | 72 | "@esbuild/android-x64@0.17.19": 73 | version "0.17.19" 74 | resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.17.19.tgz#658368ef92067866d95fb268719f98f363d13ae1" 75 | integrity sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww== 76 | 77 | "@esbuild/darwin-arm64@0.17.19": 78 | version "0.17.19" 79 | resolved "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.19.tgz" 80 | integrity sha512-80wEoCfF/hFKM6WE1FyBHc9SfUblloAWx6FJkFWTWiCoht9Mc0ARGEM47e67W9rI09YoUxJL68WHfDRYEAvOhg== 81 | 82 | "@esbuild/darwin-x64@0.17.19": 83 | version "0.17.19" 84 | resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.17.19.tgz#7751d236dfe6ce136cce343dce69f52d76b7f6cb" 85 | integrity sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw== 86 | 87 | "@esbuild/freebsd-arm64@0.17.19": 88 | version "0.17.19" 89 | resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.19.tgz#cacd171665dd1d500f45c167d50c6b7e539d5fd2" 90 | integrity sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ== 91 | 92 | "@esbuild/freebsd-x64@0.17.19": 93 | version "0.17.19" 94 | resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.17.19.tgz#0769456eee2a08b8d925d7c00b79e861cb3162e4" 95 | integrity sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ== 96 | 97 | "@esbuild/linux-arm64@0.17.19": 98 | version "0.17.19" 99 | resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.17.19.tgz#38e162ecb723862c6be1c27d6389f48960b68edb" 100 | integrity sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg== 101 | 102 | "@esbuild/linux-arm@0.17.19": 103 | version "0.17.19" 104 | resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.17.19.tgz#1a2cd399c50040184a805174a6d89097d9d1559a" 105 | integrity sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA== 106 | 107 | "@esbuild/linux-ia32@0.17.19": 108 | version "0.17.19" 109 | resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.17.19.tgz#e28c25266b036ce1cabca3c30155222841dc035a" 110 | integrity sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ== 111 | 112 | "@esbuild/linux-loong64@0.17.19": 113 | version "0.17.19" 114 | resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.17.19.tgz#0f887b8bb3f90658d1a0117283e55dbd4c9dcf72" 115 | integrity sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ== 116 | 117 | "@esbuild/linux-mips64el@0.17.19": 118 | version "0.17.19" 119 | resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.17.19.tgz#f5d2a0b8047ea9a5d9f592a178ea054053a70289" 120 | integrity sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A== 121 | 122 | "@esbuild/linux-ppc64@0.17.19": 123 | version "0.17.19" 124 | resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.17.19.tgz#876590e3acbd9fa7f57a2c7d86f83717dbbac8c7" 125 | integrity sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg== 126 | 127 | "@esbuild/linux-riscv64@0.17.19": 128 | version "0.17.19" 129 | resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.17.19.tgz#7f49373df463cd9f41dc34f9b2262d771688bf09" 130 | integrity sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA== 131 | 132 | "@esbuild/linux-s390x@0.17.19": 133 | version "0.17.19" 134 | resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.17.19.tgz#e2afd1afcaf63afe2c7d9ceacd28ec57c77f8829" 135 | integrity sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q== 136 | 137 | "@esbuild/linux-x64@0.17.19": 138 | version "0.17.19" 139 | resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.17.19.tgz#8a0e9738b1635f0c53389e515ae83826dec22aa4" 140 | integrity sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw== 141 | 142 | "@esbuild/netbsd-x64@0.17.19": 143 | version "0.17.19" 144 | resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.17.19.tgz#c29fb2453c6b7ddef9a35e2c18b37bda1ae5c462" 145 | integrity sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q== 146 | 147 | "@esbuild/openbsd-x64@0.17.19": 148 | version "0.17.19" 149 | resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.17.19.tgz#95e75a391403cb10297280d524d66ce04c920691" 150 | integrity sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g== 151 | 152 | "@esbuild/sunos-x64@0.17.19": 153 | version "0.17.19" 154 | resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.17.19.tgz#722eaf057b83c2575937d3ffe5aeb16540da7273" 155 | integrity sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg== 156 | 157 | "@esbuild/win32-arm64@0.17.19": 158 | version "0.17.19" 159 | resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.17.19.tgz#9aa9dc074399288bdcdd283443e9aeb6b9552b6f" 160 | integrity sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag== 161 | 162 | "@esbuild/win32-ia32@0.17.19": 163 | version "0.17.19" 164 | resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.17.19.tgz#95ad43c62ad62485e210f6299c7b2571e48d2b03" 165 | integrity sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw== 166 | 167 | "@esbuild/win32-x64@0.17.19": 168 | version "0.17.19" 169 | resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.17.19.tgz#8cfaf2ff603e9aabb910e9c0558c26cf32744061" 170 | integrity sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA== 171 | 172 | "@fastify/busboy@^2.0.0": 173 | version "2.1.1" 174 | resolved "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz" 175 | integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== 176 | 177 | "@jridgewell/resolve-uri@^3.0.3": 178 | version "3.1.2" 179 | resolved "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz" 180 | integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== 181 | 182 | "@jridgewell/sourcemap-codec@^1.4.10": 183 | version "1.5.0" 184 | resolved "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz" 185 | integrity sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ== 186 | 187 | "@jridgewell/trace-mapping@0.3.9": 188 | version "0.3.9" 189 | resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz" 190 | integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ== 191 | dependencies: 192 | "@jridgewell/resolve-uri" "^3.0.3" 193 | "@jridgewell/sourcemap-codec" "^1.4.10" 194 | 195 | "@types/node-forge@^1.3.0": 196 | version "1.3.11" 197 | resolved "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.11.tgz" 198 | integrity sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ== 199 | dependencies: 200 | "@types/node" "*" 201 | 202 | "@types/node@*": 203 | version "22.10.2" 204 | resolved "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz" 205 | integrity sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ== 206 | dependencies: 207 | undici-types "~6.20.0" 208 | 209 | "@types/semver@^7.5.8": 210 | version "7.5.8" 211 | resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e" 212 | integrity sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ== 213 | 214 | acorn-walk@^8.2.0: 215 | version "8.3.4" 216 | resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz" 217 | integrity sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g== 218 | dependencies: 219 | acorn "^8.11.0" 220 | 221 | acorn@^8.11.0, acorn@^8.8.0: 222 | version "8.14.0" 223 | resolved "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz" 224 | integrity sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA== 225 | 226 | as-table@^1.0.36: 227 | version "1.0.55" 228 | resolved "https://registry.npmjs.org/as-table/-/as-table-1.0.55.tgz" 229 | integrity sha512-xvsWESUJn0JN421Xb9MQw6AsMHRCUknCe0Wjlxvjud80mU4E6hQf1A6NzQKcYNmYw62MfzEtXc+badstZP3JpQ== 230 | dependencies: 231 | printable-characters "^1.0.42" 232 | 233 | blake3-wasm@^2.1.5: 234 | version "2.1.5" 235 | resolved "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz" 236 | integrity sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g== 237 | 238 | capnp-ts@^0.7.0: 239 | version "0.7.0" 240 | resolved "https://registry.npmjs.org/capnp-ts/-/capnp-ts-0.7.0.tgz" 241 | integrity sha512-XKxXAC3HVPv7r674zP0VC3RTXz+/JKhfyw94ljvF80yynK6VkTnqE3jMuN8b3dUVmmc43TjyxjW4KTsmB3c86g== 242 | dependencies: 243 | debug "^4.3.1" 244 | tslib "^2.2.0" 245 | 246 | chokidar@^4.0.1: 247 | version "4.0.3" 248 | resolved "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz" 249 | integrity sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA== 250 | dependencies: 251 | readdirp "^4.0.1" 252 | 253 | cookie@^0.7.1: 254 | version "0.7.2" 255 | resolved "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz" 256 | integrity sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w== 257 | 258 | data-uri-to-buffer@^2.0.0: 259 | version "2.0.2" 260 | resolved "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-2.0.2.tgz" 261 | integrity sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA== 262 | 263 | date-fns@^4.1.0: 264 | version "4.1.0" 265 | resolved "https://registry.npmjs.org/date-fns/-/date-fns-4.1.0.tgz" 266 | integrity sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg== 267 | 268 | debug@^4.3.1: 269 | version "4.4.0" 270 | resolved "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz" 271 | integrity sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA== 272 | dependencies: 273 | ms "^2.1.3" 274 | 275 | defu@^6.1.4: 276 | version "6.1.4" 277 | resolved "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz" 278 | integrity sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg== 279 | 280 | esbuild@0.17.19: 281 | version "0.17.19" 282 | resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.17.19.tgz" 283 | integrity sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw== 284 | optionalDependencies: 285 | "@esbuild/android-arm" "0.17.19" 286 | "@esbuild/android-arm64" "0.17.19" 287 | "@esbuild/android-x64" "0.17.19" 288 | "@esbuild/darwin-arm64" "0.17.19" 289 | "@esbuild/darwin-x64" "0.17.19" 290 | "@esbuild/freebsd-arm64" "0.17.19" 291 | "@esbuild/freebsd-x64" "0.17.19" 292 | "@esbuild/linux-arm" "0.17.19" 293 | "@esbuild/linux-arm64" "0.17.19" 294 | "@esbuild/linux-ia32" "0.17.19" 295 | "@esbuild/linux-loong64" "0.17.19" 296 | "@esbuild/linux-mips64el" "0.17.19" 297 | "@esbuild/linux-ppc64" "0.17.19" 298 | "@esbuild/linux-riscv64" "0.17.19" 299 | "@esbuild/linux-s390x" "0.17.19" 300 | "@esbuild/linux-x64" "0.17.19" 301 | "@esbuild/netbsd-x64" "0.17.19" 302 | "@esbuild/openbsd-x64" "0.17.19" 303 | "@esbuild/sunos-x64" "0.17.19" 304 | "@esbuild/win32-arm64" "0.17.19" 305 | "@esbuild/win32-ia32" "0.17.19" 306 | "@esbuild/win32-x64" "0.17.19" 307 | 308 | escape-string-regexp@^4.0.0: 309 | version "4.0.0" 310 | resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz" 311 | integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== 312 | 313 | estree-walker@^0.6.1: 314 | version "0.6.1" 315 | resolved "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz" 316 | integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w== 317 | 318 | exit-hook@^2.2.1: 319 | version "2.2.1" 320 | resolved "https://registry.npmjs.org/exit-hook/-/exit-hook-2.2.1.tgz" 321 | integrity sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw== 322 | 323 | fsevents@~2.3.2: 324 | version "2.3.3" 325 | resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz" 326 | integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== 327 | 328 | function-bind@^1.1.2: 329 | version "1.1.2" 330 | resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz" 331 | integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== 332 | 333 | get-source@^2.0.12: 334 | version "2.0.12" 335 | resolved "https://registry.npmjs.org/get-source/-/get-source-2.0.12.tgz" 336 | integrity sha512-X5+4+iD+HoSeEED+uwrQ07BOQr0kEDFMVqqpBuI+RaZBpBpHCuXxo70bjar6f0b0u/DQJsJ7ssurpP0V60Az+w== 337 | dependencies: 338 | data-uri-to-buffer "^2.0.0" 339 | source-map "^0.6.1" 340 | 341 | glob-to-regexp@^0.4.1: 342 | version "0.4.1" 343 | resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz" 344 | integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== 345 | 346 | hasown@^2.0.2: 347 | version "2.0.2" 348 | resolved "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz" 349 | integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== 350 | dependencies: 351 | function-bind "^1.1.2" 352 | 353 | is-core-module@^2.16.0: 354 | version "2.16.0" 355 | resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.0.tgz" 356 | integrity sha512-urTSINYfAYgcbLb0yDQ6egFm6h3Mo1DcF9EkyXSRjjzdHbsulg01qhwWuXdOoUBuTkbQ80KDboXa0vFJ+BDH+g== 357 | dependencies: 358 | hasown "^2.0.2" 359 | 360 | itty-time@^1.0.6: 361 | version "1.0.6" 362 | resolved "https://registry.npmjs.org/itty-time/-/itty-time-1.0.6.tgz" 363 | integrity sha512-+P8IZaLLBtFv8hCkIjcymZOp4UJ+xW6bSlQsXGqrkmJh7vSiMFSlNne0mCYagEE0N7HDNR5jJBRxwN0oYv61Rw== 364 | 365 | magic-string@^0.25.3: 366 | version "0.25.9" 367 | resolved "https://registry.npmjs.org/magic-string/-/magic-string-0.25.9.tgz" 368 | integrity sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ== 369 | dependencies: 370 | sourcemap-codec "^1.4.8" 371 | 372 | mime@^3.0.0: 373 | version "3.0.0" 374 | resolved "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz" 375 | integrity sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A== 376 | 377 | miniflare@3.20241218.0: 378 | version "3.20241218.0" 379 | resolved "https://registry.npmjs.org/miniflare/-/miniflare-3.20241218.0.tgz" 380 | integrity sha512-spYFDArH0wd+wJSTrzBrWrXJrbyJhRMJa35mat947y1jYhVV8I5V8vnD3LwjfpLr0SaEilojz1OIW7ekmnRe+w== 381 | dependencies: 382 | "@cspotcode/source-map-support" "0.8.1" 383 | acorn "^8.8.0" 384 | acorn-walk "^8.2.0" 385 | capnp-ts "^0.7.0" 386 | exit-hook "^2.2.1" 387 | glob-to-regexp "^0.4.1" 388 | stoppable "^1.1.0" 389 | undici "^5.28.4" 390 | workerd "1.20241218.0" 391 | ws "^8.18.0" 392 | youch "^3.2.2" 393 | zod "^3.22.3" 394 | 395 | ms@^2.1.3: 396 | version "2.1.3" 397 | resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" 398 | integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== 399 | 400 | mustache@^4.2.0: 401 | version "4.2.0" 402 | resolved "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz" 403 | integrity sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ== 404 | 405 | nanoid@^3.3.3: 406 | version "3.3.8" 407 | resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz" 408 | integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== 409 | 410 | node-forge@^1: 411 | version "1.3.1" 412 | resolved "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz" 413 | integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== 414 | 415 | ohash@^1.1.4: 416 | version "1.1.4" 417 | resolved "https://registry.npmjs.org/ohash/-/ohash-1.1.4.tgz" 418 | integrity sha512-FlDryZAahJmEF3VR3w1KogSEdWX3WhA5GPakFx4J81kEAiHyLMpdLLElS8n8dfNadMgAne/MywcvmogzscVt4g== 419 | 420 | path-parse@^1.0.7: 421 | version "1.0.7" 422 | resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz" 423 | integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== 424 | 425 | path-to-regexp@^6.3.0: 426 | version "6.3.0" 427 | resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz" 428 | integrity sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ== 429 | 430 | pathe@^1.1.2: 431 | version "1.1.2" 432 | resolved "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz" 433 | integrity sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ== 434 | 435 | prettier@^3.4.2: 436 | version "3.4.2" 437 | resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.4.2.tgz#a5ce1fb522a588bf2b78ca44c6e6fe5aa5a2b13f" 438 | integrity sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ== 439 | 440 | printable-characters@^1.0.42: 441 | version "1.0.42" 442 | resolved "https://registry.npmjs.org/printable-characters/-/printable-characters-1.0.42.tgz" 443 | integrity sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ== 444 | 445 | readdirp@^4.0.1: 446 | version "4.0.2" 447 | resolved "https://registry.npmjs.org/readdirp/-/readdirp-4.0.2.tgz" 448 | integrity sha512-yDMz9g+VaZkqBYS/ozoBJwaBhTbZo3UNYQHNRw1D3UFQB8oHB4uS/tAODO+ZLjGWmUbKnIlOWO+aaIiAxrUWHA== 449 | 450 | resolve@^1.22.8: 451 | version "1.22.10" 452 | resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz" 453 | integrity sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w== 454 | dependencies: 455 | is-core-module "^2.16.0" 456 | path-parse "^1.0.7" 457 | supports-preserve-symlinks-flag "^1.0.0" 458 | 459 | rollup-plugin-inject@^3.0.0: 460 | version "3.0.2" 461 | resolved "https://registry.npmjs.org/rollup-plugin-inject/-/rollup-plugin-inject-3.0.2.tgz" 462 | integrity sha512-ptg9PQwzs3orn4jkgXJ74bfs5vYz1NCZlSQMBUA0wKcGp5i5pA1AO3fOUEte8enhGUC+iapTCzEWw2jEFFUO/w== 463 | dependencies: 464 | estree-walker "^0.6.1" 465 | magic-string "^0.25.3" 466 | rollup-pluginutils "^2.8.1" 467 | 468 | rollup-plugin-node-polyfills@^0.2.1: 469 | version "0.2.1" 470 | resolved "https://registry.npmjs.org/rollup-plugin-node-polyfills/-/rollup-plugin-node-polyfills-0.2.1.tgz" 471 | integrity sha512-4kCrKPTJ6sK4/gLL/U5QzVT8cxJcofO0OU74tnB19F40cmuAKSzH5/siithxlofFEjwvw1YAhPmbvGNA6jEroA== 472 | dependencies: 473 | rollup-plugin-inject "^3.0.0" 474 | 475 | rollup-pluginutils@^2.8.1: 476 | version "2.8.2" 477 | resolved "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz" 478 | integrity sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ== 479 | dependencies: 480 | estree-walker "^0.6.1" 481 | 482 | selfsigned@^2.0.1: 483 | version "2.4.1" 484 | resolved "https://registry.npmjs.org/selfsigned/-/selfsigned-2.4.1.tgz" 485 | integrity sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q== 486 | dependencies: 487 | "@types/node-forge" "^1.3.0" 488 | node-forge "^1" 489 | 490 | semver@^7.6.3: 491 | version "7.6.3" 492 | resolved "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz" 493 | integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== 494 | 495 | source-map@^0.6.1: 496 | version "0.6.1" 497 | resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" 498 | integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== 499 | 500 | sourcemap-codec@^1.4.8: 501 | version "1.4.8" 502 | resolved "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz" 503 | integrity sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA== 504 | 505 | stacktracey@^2.1.8: 506 | version "2.1.8" 507 | resolved "https://registry.npmjs.org/stacktracey/-/stacktracey-2.1.8.tgz" 508 | integrity sha512-Kpij9riA+UNg7TnphqjH7/CzctQ/owJGNbFkfEeve4Z4uxT5+JapVLFXcsurIfN34gnTWZNJ/f7NMG0E8JDzTw== 509 | dependencies: 510 | as-table "^1.0.36" 511 | get-source "^2.0.12" 512 | 513 | stoppable@^1.1.0: 514 | version "1.1.0" 515 | resolved "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz" 516 | integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw== 517 | 518 | supports-preserve-symlinks-flag@^1.0.0: 519 | version "1.0.0" 520 | resolved "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz" 521 | integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== 522 | 523 | tslib@^2.2.0: 524 | version "2.8.1" 525 | resolved "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz" 526 | integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== 527 | 528 | ufo@^1.5.4: 529 | version "1.5.4" 530 | resolved "https://registry.npmjs.org/ufo/-/ufo-1.5.4.tgz" 531 | integrity sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ== 532 | 533 | undici-types@~6.20.0: 534 | version "6.20.0" 535 | resolved "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz" 536 | integrity sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg== 537 | 538 | undici@^5.28.4: 539 | version "5.28.4" 540 | resolved "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz" 541 | integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== 542 | dependencies: 543 | "@fastify/busboy" "^2.0.0" 544 | 545 | "unenv@npm:unenv-nightly@2.0.0-20241204-140205-a5d5190": 546 | version "2.0.0-20241204-140205-a5d5190" 547 | resolved "https://registry.npmjs.org/unenv-nightly/-/unenv-nightly-2.0.0-20241204-140205-a5d5190.tgz" 548 | integrity sha512-jpmAytLeiiW01pl5bhVn9wYJ4vtiLdhGe10oXlJBuQEX8mxjxO8BlEXGHU4vr4yEikjFP1wsomTHt/CLU8kUwg== 549 | dependencies: 550 | defu "^6.1.4" 551 | ohash "^1.1.4" 552 | pathe "^1.1.2" 553 | ufo "^1.5.4" 554 | 555 | workerd@1.20241218.0: 556 | version "1.20241218.0" 557 | resolved "https://registry.npmjs.org/workerd/-/workerd-1.20241218.0.tgz" 558 | integrity sha512-7Z3D4vOVChMz9mWDffE299oQxUWm/pbkeAWx1btVamPcAK/2IuoNBhwflWo3jyuKuxvYuFAdIucgYxc8ICqXiA== 559 | optionalDependencies: 560 | "@cloudflare/workerd-darwin-64" "1.20241218.0" 561 | "@cloudflare/workerd-darwin-arm64" "1.20241218.0" 562 | "@cloudflare/workerd-linux-64" "1.20241218.0" 563 | "@cloudflare/workerd-linux-arm64" "1.20241218.0" 564 | "@cloudflare/workerd-windows-64" "1.20241218.0" 565 | 566 | wrangler@^3.99.0: 567 | version "3.99.0" 568 | resolved "https://registry.npmjs.org/wrangler/-/wrangler-3.99.0.tgz" 569 | integrity sha512-k0x4rT3G/QCbxcoZY7CHRVlAIS8WMmKdga6lf4d2c3gXFqssh44vwlTDuARA9QANBxKJTcA7JPTJRfUDhd9QBA== 570 | dependencies: 571 | "@cloudflare/kv-asset-handler" "0.3.4" 572 | "@esbuild-plugins/node-globals-polyfill" "^0.2.3" 573 | "@esbuild-plugins/node-modules-polyfill" "^0.2.2" 574 | blake3-wasm "^2.1.5" 575 | chokidar "^4.0.1" 576 | date-fns "^4.1.0" 577 | esbuild "0.17.19" 578 | itty-time "^1.0.6" 579 | miniflare "3.20241218.0" 580 | nanoid "^3.3.3" 581 | path-to-regexp "^6.3.0" 582 | resolve "^1.22.8" 583 | selfsigned "^2.0.1" 584 | source-map "^0.6.1" 585 | unenv "npm:unenv-nightly@2.0.0-20241204-140205-a5d5190" 586 | workerd "1.20241218.0" 587 | xxhash-wasm "^1.0.1" 588 | optionalDependencies: 589 | fsevents "~2.3.2" 590 | 591 | ws@^8.18.0: 592 | version "8.18.0" 593 | resolved "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz" 594 | integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== 595 | 596 | xxhash-wasm@^1.0.1: 597 | version "1.1.0" 598 | resolved "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz" 599 | integrity sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA== 600 | 601 | youch@^3.2.2: 602 | version "3.3.4" 603 | resolved "https://registry.npmjs.org/youch/-/youch-3.3.4.tgz" 604 | integrity sha512-UeVBXie8cA35DS6+nBkls68xaBBXCye0CNznrhszZjTbRVnJKQuNsyLKBTTL4ln1o1rh2PKtv35twV7irj5SEg== 605 | dependencies: 606 | cookie "^0.7.1" 607 | mustache "^4.2.0" 608 | stacktracey "^2.1.8" 609 | 610 | zod@^3.22.3: 611 | version "3.24.1" 612 | resolved "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz" 613 | integrity sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A== 614 | -------------------------------------------------------------------------------- /patches/duckdb+1.0.0.patch: -------------------------------------------------------------------------------- 1 | diff -ruN packages/vendor/duckdb/Cargo.toml packages/vendor/duckdb/Cargo.toml 2 | --- a/Cargo.toml 1969-12-31 18:00:01 3 | +++ b/Cargo.toml 2024-09-11 01:33:53 4 | @@ -170,18 +170,10 @@ 5 | 6 | [features] 7 | appender-arrow = ["vtab-arrow"] 8 | -buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"] 9 | -bundled = ["libduckdb-sys/bundled"] 10 | default = [] 11 | extensions-full = [ 12 | - "json", 13 | - "parquet", 14 | "vtab-full", 15 | ] 16 | -json = [ 17 | - "libduckdb-sys/json", 18 | - "bundled", 19 | -] 20 | modern-full = [ 21 | "chrono", 22 | "serde_json", 23 | @@ -190,10 +182,6 @@ 24 | "uuid", 25 | "polars", 26 | ] 27 | -parquet = [ 28 | - "libduckdb-sys/parquet", 29 | - "bundled", 30 | -] 31 | polars = ["dep:polars"] 32 | vtab = [] 33 | vtab-arrow = [ 34 | @@ -212,4 +200,7 @@ 35 | vtab-loadable = [ 36 | "vtab", 37 | "duckdb-loadable-macros", 38 | -] 39 | +] 40 | +loadable_extension = [ 41 | + "libduckdb-sys/loadable_extension" 42 | +] 43 | \ No newline at end of file 44 | diff -ruN packages/vendor/duckdb/src/vtab/logical_type.rs packages/vendor/duckdb/src/vtab/logical_type.rs 45 | --- a/src/vtab/logical_type.rs 2006-07-23 20:21:28 46 | +++ b/src/vtab/logical_type.rs 2024-09-11 11:55:25 47 | @@ -164,6 +164,27 @@ 48 | } 49 | } 50 | 51 | + pub fn enumeration(names: &[&str]) -> Self { 52 | + let strings: Vec = names 53 | + .iter() 54 | + .map(|s| CString::new(*s).unwrap()) 55 | + .collect(); 56 | + 57 | + let mut pointers: Vec<*const ::std::os::raw::c_char> = strings 58 | + .iter() 59 | + .map(|cstr| cstr.as_ptr()) 60 | + .collect(); 61 | + 62 | + unsafe { 63 | + Self { 64 | + ptr: duckdb_create_enum_type( 65 | + pointers.as_mut_ptr(), 66 | + names.len() as _, 67 | + ) 68 | + } 69 | + } 70 | + } 71 | + 72 | /// Creates a map type from its child type. 73 | pub fn map(key: &LogicalType, value: &LogicalType) -> Self { 74 | unsafe { 75 | diff -ruN packages/vendor/duckdb/src/vtab/mod.rs packages/vendor/duckdb/src/vtab/mod.rs 76 | --- a/src/vtab/mod.rs 2006-07-23 20:21:28 77 | +++ b/src/vtab/mod.rs 2024-09-11 01:33:53 78 | @@ -148,6 +148,19 @@ 79 | } 80 | } 81 | 82 | +unsafe extern "C" fn local_init(info: duckdb_init_info) 83 | +where 84 | + T: VTabLocalData, 85 | +{ 86 | + let info = InitInfo::from(info); 87 | + let data = malloc_data_c::(); 88 | + let result = T::local_init(&info, data); 89 | + info.set_init_data(data.cast(), Some(drop_data_c::)); 90 | + if result.is_err() { 91 | + info.set_error(&result.err().unwrap().to_string()); 92 | + } 93 | +} 94 | + 95 | unsafe extern "C" fn bind(info: duckdb_bind_info) 96 | where 97 | T: VTab, 98 | @@ -162,26 +175,21 @@ 99 | } 100 | 101 | impl Connection { 102 | - /// Register the given TableFunction with the current db 103 | - #[inline] 104 | pub fn register_table_function(&self, name: &str) -> Result<()> { 105 | - let table_function = TableFunction::default(); 106 | - table_function 107 | - .set_name(name) 108 | - .supports_pushdown(T::supports_pushdown()) 109 | - .set_bind(Some(bind::)) 110 | - .set_init(Some(init::)) 111 | - .set_function(Some(func::)); 112 | - for ty in T::parameters().unwrap_or_default() { 113 | - table_function.add_parameter(&ty); 114 | - } 115 | - for (name, ty) in T::named_parameters().unwrap_or_default() { 116 | - table_function.add_named_parameter(&name, &ty); 117 | - } 118 | + let table_function = into_table_function::(); 119 | + table_function.set_name(name); 120 | self.db.borrow_mut().register_table_function(table_function) 121 | } 122 | + 123 | + #[inline] 124 | + pub fn register_table_function_local_init(&self, name: &str) -> Result<()> { 125 | + let table_function = into_table_function_local_init::(); 126 | + table_function.set_name(name); 127 | + self.db.borrow_mut().register_table_function(table_function) 128 | + } 129 | } 130 | 131 | + 132 | impl InnerConnection { 133 | /// Register the given TableFunction with the current db 134 | pub fn register_table_function(&mut self, table_function: TableFunction) -> Result<()> { 135 | @@ -193,6 +201,35 @@ 136 | } 137 | Ok(()) 138 | } 139 | +} 140 | + 141 | +fn into_table_function() -> TableFunction { 142 | + let table_function = TableFunction::default(); 143 | + table_function 144 | + .supports_pushdown(T::supports_pushdown()) 145 | + .set_bind(Some(bind::)) 146 | + .set_init(Some(init::)) 147 | + .set_function(Some(func::)); 148 | + for ty in T::parameters().unwrap_or_default() { 149 | + table_function.add_parameter(&ty); 150 | + } 151 | + for (name, ty) in T::named_parameters().unwrap_or_default() { 152 | + table_function.add_named_parameter(&name, &ty); 153 | + } 154 | + 155 | + table_function 156 | +} 157 | + 158 | +pub trait VTabLocalData: VTab { 159 | + type LocalInitData: Sized + Free; 160 | + 161 | + fn local_init(init: &InitInfo, data: *mut Self::LocalInitData) -> Result<(), Box>; 162 | +} 163 | + 164 | +fn into_table_function_local_init() -> TableFunction { 165 | + let table_function = into_table_function::(); 166 | + table_function.set_local_init(Some(local_init::)); 167 | + table_function 168 | } 169 | 170 | #[cfg(test)] 171 | -------------------------------------------------------------------------------- /patches/duckdb-loadable-macros+0.1.2.patch: -------------------------------------------------------------------------------- 1 | diff -ruN packages/backup/duckdb-loadable-macros/Cargo.toml packages/vendor/duckdb-loadable-macros/Cargo.toml 2 | --- a/Cargo.toml 1969-12-31 18:00:01 3 | +++ b/Cargo.toml 2024-09-05 00:42:50 4 | @@ -46,3 +46,6 @@ 5 | "fold", 6 | "parsing", 7 | ] 8 | + 9 | +[dependencies.darling] 10 | +version = "0.20.10" 11 | diff -ruN packages/backup/duckdb-loadable-macros/src/lib.rs packages/vendor/duckdb-loadable-macros/src/lib.rs 12 | --- a/src/lib.rs 2006-07-23 20:21:28 13 | +++ b/src/lib.rs 2024-09-05 02:30:06 14 | @@ -1,25 +1,100 @@ 15 | #![allow(clippy::redundant_clone)] 16 | -use proc_macro2::{Ident, Span}; 17 | +use proc_macro2::{Ident, Literal, Punct, Span}; 18 | 19 | use syn::{parse_macro_input, spanned::Spanned, Item}; 20 | 21 | use proc_macro::TokenStream; 22 | use quote::quote_spanned; 23 | +use std::ffi::{CStr, CString}; 24 | 25 | +use darling::{ast::NestedMeta, Error, FromMeta}; 26 | +use syn::ItemFn; 27 | + 28 | +/// For parsing the arguments to the duckdb_entrypoint_c_api macro 29 | +#[derive(Debug, FromMeta)] 30 | +struct CEntryPointMacroArgs { 31 | + #[darling(default)] 32 | + ext_name: String, 33 | + min_duckdb_version: Option, 34 | +} 35 | + 36 | /// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name. 37 | #[proc_macro_attribute] 38 | +pub fn duckdb_entrypoint_c_api(attr: TokenStream, item: TokenStream) -> TokenStream { 39 | + let attr_args = match NestedMeta::parse_meta_list(attr.into()) { 40 | + Ok(v) => v, 41 | + Err(e) => { 42 | + return TokenStream::from(Error::from(e).write_errors()); 43 | + } 44 | + }; 45 | + 46 | + let args = match CEntryPointMacroArgs::from_list(&attr_args) { 47 | + Ok(v) => v, 48 | + Err(e) => { 49 | + return TokenStream::from(e.write_errors()); 50 | + } 51 | + }; 52 | + 53 | + let minimum_duckdb_version = match args.min_duckdb_version { 54 | + Some(i) => CString::new(i).unwrap(), 55 | + None => CString::from(c"dev"), 56 | + }; 57 | + 58 | + let minimum_duckdb_version_literal = Literal::c_string(minimum_duckdb_version.as_c_str()); 59 | + 60 | + let ast = parse_macro_input!(item as syn::Item); 61 | + 62 | + match ast { 63 | + Item::Fn(mut func) => { 64 | + let c_entrypoint = Ident::new( 65 | + format!("{}_init_c_api", args.ext_name).as_str(), 66 | + Span::call_site(), 67 | + ); 68 | + let original_funcname = func.sig.ident.to_string(); 69 | + let prefixed_original_function = func.sig.ident.clone(); 70 | + 71 | + quote_spanned! {func.span()=> 72 | + 73 | + /// # Safety 74 | + /// 75 | + /// Will be called by duckdb 76 | + #[no_mangle] 77 | + pub unsafe extern "C" fn #c_entrypoint(info: ffi::duckdb_extension_info, access: *const ffi::duckdb_extension_access) { 78 | + ffi::duckdb_rs_extension_api_init(info, *access, #minimum_duckdb_version_literal).expect("Failed to initialize DuckDB C Extension API"); 79 | + 80 | + let db : ffi::duckdb_database = *(*access).get_database.unwrap()(info); 81 | + let connection = Connection::open_from_raw(db.cast()).expect("can't open db connection"); 82 | + #prefixed_original_function(connection).expect("init failed"); 83 | + } 84 | + 85 | + #func 86 | + } 87 | + .into() 88 | + } 89 | + _ => panic!("Only function items are allowed on duckdb_entrypoint"), 90 | + } 91 | +} 92 | + 93 | +/// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name. 94 | +#[proc_macro_attribute] 95 | pub fn duckdb_entrypoint(_attr: TokenStream, item: TokenStream) -> TokenStream { 96 | let ast = parse_macro_input!(item as syn::Item); 97 | match ast { 98 | Item::Fn(mut func) => { 99 | let c_entrypoint = func.sig.ident.clone(); 100 | let c_entrypoint_version = Ident::new( 101 | - c_entrypoint.to_string().replace("_init", "_version").as_str(), 102 | + c_entrypoint 103 | + .to_string() 104 | + .replace("_init", "_version") 105 | + .as_str(), 106 | Span::call_site(), 107 | ); 108 | 109 | let original_funcname = func.sig.ident.to_string(); 110 | - func.sig.ident = Ident::new(format!("_{}", original_funcname).as_str(), func.sig.ident.span()); 111 | + func.sig.ident = Ident::new( 112 | + format!("_{}", original_funcname).as_str(), 113 | + func.sig.ident.span(), 114 | + ); 115 | 116 | let prefixed_original_function = func.sig.ident.clone(); 117 | 118 | --------------------------------------------------------------------------------