├── .editorconfig ├── ts ├── .prettierrc ├── pnpm-workspace.yaml ├── .prettierignore ├── .gitignore ├── pkgs │ ├── duckdb-ui-client │ │ ├── src │ │ │ ├── index.ts │ │ │ ├── serialization │ │ │ │ ├── types │ │ │ │ │ ├── TokenizeResult.ts │ │ │ │ │ ├── DataChunk.ts │ │ │ │ │ ├── ColumnNamesAndTypes.ts │ │ │ │ │ ├── QueryResult.ts │ │ │ │ │ ├── Vector.ts │ │ │ │ │ └── TypeInfo.ts │ │ │ │ ├── functions │ │ │ │ │ ├── tokenizeResultFromBuffer.ts │ │ │ │ │ ├── deserializeFromBuffer.ts │ │ │ │ │ ├── basicReaders.ts │ │ │ │ │ ├── resultReaders.ts │ │ │ │ │ └── typeReaders.ts │ │ │ │ ├── constants │ │ │ │ │ └── LogicalTypeId.ts │ │ │ │ └── classes │ │ │ │ │ ├── BinaryStreamReader.ts │ │ │ │ │ └── BinaryDeserializer.ts │ │ │ ├── tsconfig.json │ │ │ ├── util │ │ │ │ └── functions │ │ │ │ │ ├── randomString.ts │ │ │ │ │ └── toBase64.ts │ │ │ ├── conversion │ │ │ │ └── functions │ │ │ │ │ ├── isRowValid.ts │ │ │ │ │ ├── vectorGetters.ts │ │ │ │ │ ├── dataViewReaders.ts │ │ │ │ │ └── typeInfoGetters.ts │ │ │ ├── http │ │ │ │ └── functions │ │ │ │ │ ├── sendDuckDBUIHttpRequest.ts │ │ │ │ │ └── makeDuckDBUIHttpRequestHeaders.ts │ │ │ ├── client │ │ │ │ ├── types │ │ │ │ │ ├── DuckDBUIRunOptions.ts │ │ │ │ │ └── MaterializedRunResult.ts │ │ │ │ ├── functions │ │ │ │ │ └── materializedRunResultFromQueueResult.ts │ │ │ │ └── classes │ │ │ │ │ ├── DuckDBUIClient.ts │ │ │ │ │ └── DuckDBUIClientConnection.ts │ │ │ └── data-chunk │ │ │ │ └── classes │ │ │ │ ├── DuckDBDataChunk.ts │ │ │ │ └── DuckDBDataChunkIterator.ts │ │ ├── test │ │ │ ├── tsconfig.json │ │ │ ├── helpers │ │ │ │ ├── makeBuffer.ts │ │ │ │ └── mockRequests.ts │ │ │ ├── util │ │ │ │ └── functions │ │ │ │ │ ├── toBase64.test.ts │ │ │ │ │ └── randomString.test.ts │ │ │ ├── serialization │ │ │ │ └── classes │ │ │ │ │ ├── BinaryStreamReader.test.ts │ │ │ │ │ └── BinaryDeserializer.test.ts │ │ │ └── http │ │ │ │ ├── functions │ │ │ │ ├── makeDuckDBUIHttpRequestHeaders.test.ts │ │ │ │ └── sendDuckDBUIHttpRequest.test.ts │ │ │ │ └── classes │ │ │ │ └── DuckDBUIHttpRequestQueue.test.ts │ │ └── package.json │ ├── duckdb-data-values │ │ ├── src │ │ │ ├── DuckDBToStringOptions.ts │ │ │ ├── Json.ts │ │ │ ├── tsconfig.json │ │ │ ├── DuckDBMapEntry.ts │ │ │ ├── DuckDBStructEntry.ts │ │ │ ├── DuckDBValue.ts │ │ │ ├── conversion │ │ │ │ ├── displayStringForDuckDBValue.ts │ │ │ │ ├── jsonFromDuckDBValue.ts │ │ │ │ ├── hexFromBlob.ts │ │ │ │ ├── stringFromBlob.ts │ │ │ │ ├── duckDBValueToSql.ts │ │ │ │ ├── getBigNumFromBytes.ts │ │ │ │ └── stringFromDecimal.ts │ │ │ ├── SpecialDuckDBValue.ts │ │ │ ├── DuckDBBlobValue.ts │ │ │ ├── DuckDBDateValue.ts │ │ │ ├── DuckDBTimeValue.ts │ │ │ ├── DuckDBTimestampSecondsValue.ts │ │ │ ├── DuckDBTimestampNanosecondsValue.ts │ │ │ ├── DuckDBTimestampMillisecondsValue.ts │ │ │ ├── DuckDBTimestampMicrosecondsValue.ts │ │ │ ├── DuckDBIntervalValue.ts │ │ │ ├── DuckDBTimestampTZValue.ts │ │ │ ├── DuckDBListValue.ts │ │ │ ├── DuckDBArrayValue.ts │ │ │ ├── DuckDBDecimalValue.ts │ │ │ ├── DuckDBTimeTZValue.ts │ │ │ ├── DuckDBMapValue.ts │ │ │ ├── DuckDBStructValue.ts │ │ │ ├── index.ts │ │ │ ├── DuckDBUUIDValue.ts │ │ │ └── DuckDBBitValue.ts │ │ ├── test │ │ │ ├── tsconfig.json │ │ │ ├── conversion │ │ │ │ ├── jsonFromDuckDBValue.test.ts │ │ │ │ ├── displayStringForDuckDBValue.test.ts │ │ │ │ ├── duckDBValueToSql.test.ts │ │ │ │ └── getBigNumFromBytes.test.ts │ │ │ ├── DuckDBTimeValue.test.ts │ │ │ ├── DuckDBDateValue.test.ts │ │ │ ├── DuckDBTimestampNanosecondsValue.test.ts │ │ │ ├── DuckDBBitValue.test.ts │ │ │ ├── DuckDBUUIDValue.test.ts │ │ │ ├── DuckDBTimestampSecondsValue.test.ts │ │ │ ├── DuckDBTimestampMillisecondsValue.test.ts │ │ │ ├── DuckDBArrayValue.test.ts │ │ │ ├── DuckDBListValue.test.ts │ │ │ ├── DuckDBTimeTZValue.test.ts │ │ │ ├── DuckDBTimestampMicrosecondsValue.test.ts │ │ │ ├── DuckDBBlobValue.test.ts │ │ │ └── DuckDBMapValue.test.ts │ │ └── package.json │ ├── duckdb-data-types │ │ ├── test │ │ │ └── tsconfig.json │ │ ├── src │ │ │ ├── tsconfig.json │ │ │ ├── index.ts │ │ │ ├── sql.ts │ │ │ ├── DuckDBTypeId.ts │ │ │ └── extensionTypes.ts │ │ └── package.json │ └── duckdb-data-reader │ │ ├── test │ │ └── tsconfig.json │ │ ├── src │ │ ├── tsconfig.json │ │ ├── DuckDBRow.ts │ │ ├── index.ts │ │ ├── AsyncDuckDBDataBatchIterator.ts │ │ ├── MemoryDuckDBData.ts │ │ ├── ColumnFilteredDuckDBData.ts │ │ └── DuckDBData.ts │ │ └── package.json ├── tsconfig.base.json ├── tsconfig.library.json ├── tsconfig.test.json ├── tsconfig.json ├── eslint.config.mjs ├── package.json └── README.md ├── vcpkg.json ├── test ├── sql │ └── ui.test └── README.md ├── .gitignore ├── src ├── include │ ├── utils │ │ ├── encoding.hpp │ │ ├── md_helpers.hpp │ │ ├── env.hpp │ │ ├── serialization.hpp │ │ └── helpers.hpp │ ├── version.hpp │ ├── ui_extension.hpp │ ├── watcher.hpp │ ├── state.hpp │ ├── event_dispatcher.hpp │ ├── settings.hpp │ └── http_server.hpp ├── settings.cpp ├── utils │ ├── env.cpp │ ├── md_helpers.cpp │ ├── helpers.cpp │ ├── encoding.cpp │ └── serialization.cpp ├── state.cpp ├── event_dispatcher.cpp └── watcher.cpp ├── .gitmodules ├── Makefile ├── extension_config.cmake ├── scripts ├── setup-custom-toolchain.sh └── extension-upload.sh ├── LICENSE ├── third_party └── httplib │ └── LICENSE ├── .github └── workflows │ ├── TypeScriptWorkspace.yml │ └── MainDistributionPipeline.yml ├── docs └── UPDATING.md ├── CMakeLists.txt └── README.md /.editorconfig: -------------------------------------------------------------------------------- 1 | duckdb/.editorconfig -------------------------------------------------------------------------------- /ts/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true 3 | } 4 | -------------------------------------------------------------------------------- /ts/pnpm-workspace.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | - 'pkgs/*' 3 | -------------------------------------------------------------------------------- /ts/.prettierignore: -------------------------------------------------------------------------------- 1 | out 2 | **/*.json 3 | README.md 4 | -------------------------------------------------------------------------------- /vcpkg.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": [ 3 | "openssl" 4 | ] 5 | } -------------------------------------------------------------------------------- /ts/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | **/out/* 3 | **/test/tsconfig.tsbuildinfo 4 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './client/classes/DuckDBUIClient.js'; 2 | -------------------------------------------------------------------------------- /test/sql/ui.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/ui.test 2 | # description: test ui extension 3 | # group: [ui] 4 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBToStringOptions.ts: -------------------------------------------------------------------------------- 1 | export interface DuckDBToStringOptions { 2 | timeZone?: string; 3 | } 4 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/types/TokenizeResult.ts: -------------------------------------------------------------------------------- 1 | export interface TokenizeResult { 2 | offsets: number[]; 3 | types: number[]; 4 | } 5 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-types/test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.test.json", 3 | "references": [ 4 | { "path": "../src" } 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.test.json", 3 | "references": [ 4 | { "path": "../src" } 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-reader/test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.test.json", 3 | "references": [ 4 | { "path": "../src" } 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-types/src/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.library.json", 3 | "compilerOptions": { 4 | "outDir": "../out" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/Json.ts: -------------------------------------------------------------------------------- 1 | export type Json = 2 | | null 3 | | boolean 4 | | number 5 | | string 6 | | Json[] 7 | | { [key: string]: Json }; 8 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.test.json", 3 | "references": [ 4 | { "path": "../src" } 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.library.json", 3 | "compilerOptions": { 4 | "outDir": "../out" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .idea 3 | cmake-build-debug 4 | duckdb_unittest_tempdir/ 5 | .DS_Store 6 | testext 7 | test/python/__pycache__/ 8 | .Rhistory 9 | .vscode/ 10 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-reader/src/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.library.json", 3 | "compilerOptions": { 4 | "outDir": "../out" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.library.json", 3 | "compilerOptions": { 4 | "outDir": "../out" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-reader/src/DuckDBRow.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBValue } from '@duckdb/data-values'; 2 | 3 | export interface DuckDBRow { 4 | readonly [columnName: string]: DuckDBValue; 5 | } 6 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/types/DataChunk.ts: -------------------------------------------------------------------------------- 1 | import { Vector } from './Vector.js'; 2 | 3 | export interface DataChunk { 4 | rowCount: number; 5 | vectors: Vector[]; 6 | } 7 | -------------------------------------------------------------------------------- /src/include/utils/encoding.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace duckdb { 6 | 7 | std::string DecodeBase64(const std::string &str); 8 | 9 | } // namespace duckdb 10 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-types/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './DuckDBType.js'; 2 | export * from './DuckDBTypeId.js'; 3 | export * from './extensionTypes.js'; 4 | export * from './parseLogicalTypeString.js'; 5 | -------------------------------------------------------------------------------- /src/include/utils/md_helpers.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace duckdb { 6 | bool IsMDConnected(Connection &); 7 | std::string GetMDToken(Connection &); 8 | } // namespace duckdb 9 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBMapEntry.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBValue } from './DuckDBValue.js'; 2 | 3 | export interface DuckDBMapEntry { 4 | readonly key: DuckDBValue; 5 | readonly value: DuckDBValue; 6 | } 7 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBStructEntry.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBValue } from './DuckDBValue.js'; 2 | 3 | export interface DuckDBStructEntry { 4 | readonly key: string; 5 | readonly value: DuckDBValue; 6 | } 7 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/types/ColumnNamesAndTypes.ts: -------------------------------------------------------------------------------- 1 | import { TypeIdAndInfo } from './TypeInfo.js'; 2 | 3 | export interface ColumnNamesAndTypes { 4 | names: string[]; 5 | types: TypeIdAndInfo[]; 6 | } 7 | -------------------------------------------------------------------------------- /ts/tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "forceConsistentCasingInFileNames": true, 4 | "noFallthroughCasesInSwitch": true, 5 | "noUnusedLocals": true, 6 | "noUnusedParameters": true, 7 | "strict": true 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "duckdb"] 2 | path = duckdb 3 | url = https://github.com/duckdb/duckdb 4 | branch = v1.4.0 5 | [submodule "extension-ci-tools"] 6 | path = extension-ci-tools 7 | url = https://github.com/duckdb/extension-ci-tools 8 | branch = v1.4.0 9 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-types/src/sql.ts: -------------------------------------------------------------------------------- 1 | export function quotedString(input: string): string { 2 | return `'${input.replace(`'`, `''`)}'`; 3 | } 4 | 5 | export function quotedIdentifier(input: string): string { 6 | return `"${input.replace(`"`, `""`)}"`; 7 | } 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) 2 | 3 | # Configuration of extension 4 | EXT_NAME=ui 5 | EXT_CONFIG=${PROJ_DIR}extension_config.cmake 6 | 7 | # Include the Makefile from extension-ci-tools 8 | include extension-ci-tools/makefiles/duckdb_extension.Makefile -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-reader/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './AsyncDuckDBDataBatchIterator.js'; 2 | export * from './ColumnFilteredDuckDBData.js'; 3 | export * from './DuckDBData.js'; 4 | export * from './DuckDBDataReader.js'; 5 | export * from './DuckDBRow.js'; 6 | export * from './MemoryDuckDBData.js'; 7 | -------------------------------------------------------------------------------- /ts/tsconfig.library.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.base.json", 3 | "compilerOptions": { 4 | "composite": true, 5 | "declaration": true, 6 | "esModuleInterop": true, 7 | "module": "NodeNext", 8 | "moduleResolution": "NodeNext", 9 | "target": "ESNext", 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBValue.ts: -------------------------------------------------------------------------------- 1 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 2 | 3 | export type DuckDBValue = 4 | | null 5 | | boolean 6 | | number 7 | | string 8 | | bigint // TODO: Should types requiring bigint be SpecialDBValues? 9 | | SpecialDuckDBValue; 10 | -------------------------------------------------------------------------------- /src/include/version.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifndef UI_EXTENSION_SEQ_NUM 4 | #error "UI_EXTENSION_SEQ_NUM must be defined" 5 | #endif 6 | #ifndef UI_EXTENSION_GIT_SHA 7 | #error "UI_EXTENSION_GIT_SHA must be defined" 8 | #endif 9 | #define UI_EXTENSION_VERSION UI_EXTENSION_SEQ_NUM "-" UI_EXTENSION_GIT_SHA 10 | -------------------------------------------------------------------------------- /extension_config.cmake: -------------------------------------------------------------------------------- 1 | # This file is included by DuckDB's build system. It specifies which extension to load 2 | 3 | # Extension from this repo 4 | duckdb_extension_load(ui 5 | SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} 6 | LOAD_TESTS 7 | ) 8 | 9 | # Any extra extensions that should be built 10 | # e.g.: duckdb_extension_load(json) -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/helpers/makeBuffer.ts: -------------------------------------------------------------------------------- 1 | export function makeBuffer(bytes: number[]): ArrayBuffer { 2 | const buffer = new ArrayBuffer(bytes.length); 3 | const dv = new DataView(buffer); 4 | for (let offset = 0; offset < bytes.length; offset++) { 5 | dv.setUint8(offset, bytes[offset]); 6 | } 7 | return buffer; 8 | } 9 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/util/functions/randomString.ts: -------------------------------------------------------------------------------- 1 | export function randomString( 2 | length: number = 12, 3 | chars: string = '$0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz', 4 | ): string { 5 | return Array.from({ length }) 6 | .map((_) => chars[Math.floor(Math.random() * chars.length)]) 7 | .join(''); 8 | } 9 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-reader/src/AsyncDuckDBDataBatchIterator.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBData } from './DuckDBData.js'; 2 | 3 | export type DuckDBDataBatchIteratorResult = IteratorResult< 4 | DuckDBData, 5 | DuckDBData | undefined 6 | >; 7 | 8 | export type AsyncDuckDBDataBatchIterator = AsyncIterator< 9 | DuckDBData, 10 | DuckDBData | undefined 11 | >; 12 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/conversion/functions/isRowValid.ts: -------------------------------------------------------------------------------- 1 | import { getUInt64 } from './dataViewReaders.js'; 2 | 3 | export function isRowValid(validity: DataView | null, row: number): boolean { 4 | if (!validity) return true; 5 | const bigint = getUInt64(validity, Math.floor(row / 64) * 8); 6 | return (bigint & (1n << BigInt(row % 64))) !== 0n; 7 | } 8 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/conversion/displayStringForDuckDBValue.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBValue } from '../DuckDBValue.js'; 2 | 3 | export function displayStringForDuckDBValue(value: DuckDBValue): string { 4 | if (value == null) { 5 | return 'NULL'; 6 | } 7 | if (typeof value === 'string') { 8 | return `'${value.replace(/'/g, "''")}'`; 9 | } 10 | return String(value); 11 | } 12 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/http/functions/sendDuckDBUIHttpRequest.ts: -------------------------------------------------------------------------------- 1 | export async function sendDuckDBUIHttpRequest( 2 | url: string, 3 | body: string, 4 | headers?: Headers, 5 | ): Promise { 6 | const response = await fetch(url, { 7 | method: 'POST', 8 | headers, 9 | body, 10 | }); 11 | const buffer = await response.arrayBuffer(); 12 | return buffer; 13 | } 14 | -------------------------------------------------------------------------------- /src/include/utils/env.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace duckdb { 7 | 8 | const char *TryGetEnv(const char *name); 9 | 10 | std::string GetEnvOrDefault(const char *name, const char *default_value); 11 | 12 | uint32_t GetEnvOrDefaultInt(const char *name, uint32_t default_value); 13 | 14 | bool IsEnvEnabled(const char *name); 15 | 16 | } // namespace duckdb 17 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/util/functions/toBase64.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { toBase64 } from '../../../src/util/functions/toBase64'; 3 | 4 | suite('toBase64', () => { 5 | test('basic', () => { 6 | expect(atob(toBase64('duck'))).toBe('duck'); 7 | }); 8 | test('unicode', () => { 9 | expect(atob(toBase64('🦆'))).toBe('\xF0\x9F\xA6\x86'); 10 | }); 11 | }); 12 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/client/types/DuckDBUIRunOptions.ts: -------------------------------------------------------------------------------- 1 | export interface DuckDBUIRunOptions { 2 | description?: string; 3 | databaseName?: string; 4 | schemaName?: string; 5 | errorsAsJson?: boolean; 6 | parameters?: unknown[]; 7 | resultRowLimit?: number; 8 | resultDatabaseName?: string; 9 | resultSchemaName?: string; 10 | resultTableName?: string; 11 | resultTableRowLimit?: number; 12 | } 13 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/helpers/mockRequests.ts: -------------------------------------------------------------------------------- 1 | import { RequestHandler } from 'msw'; 2 | import { setupServer } from 'msw/node'; 3 | 4 | export async function mockRequests( 5 | handlers: RequestHandler[], 6 | func: () => Promise, 7 | ) { 8 | const server = setupServer(...handlers); 9 | try { 10 | server.listen(); 11 | await func(); 12 | } finally { 13 | server.close(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/include/ui_extension.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "duckdb.hpp" 4 | 5 | namespace duckdb { 6 | 7 | class UiExtension : public Extension { 8 | public: 9 | #ifdef DUCKDB_CPP_EXTENSION_ENTRY 10 | void Load(ExtensionLoader &loader) override; 11 | #else 12 | void Load(DuckDB &db) override; 13 | #endif 14 | 15 | std::string Name() override; 16 | std::string Version() const override; 17 | }; 18 | 19 | } // namespace duckdb 20 | -------------------------------------------------------------------------------- /ts/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.base.json", 3 | "compilerOptions": { 4 | "allowImportingTsExtensions": true, 5 | "esModuleInterop": true, 6 | "isolatedModules": true, 7 | "module": "ESNext", 8 | "moduleResolution": "Bundler", 9 | "noEmit": true, 10 | "resolveJsonModule": true, 11 | "skipLibCheck": true, 12 | "target": "ESNext", 13 | "useDefineForClassFields": true, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/functions/tokenizeResultFromBuffer.ts: -------------------------------------------------------------------------------- 1 | import { TokenizeResult } from '../types/TokenizeResult.js'; 2 | import { deserializerFromBuffer } from './deserializeFromBuffer.js'; 3 | import { readTokenizeResult } from './resultReaders.js'; 4 | 5 | export function tokenizeResultFromBuffer(buffer: ArrayBuffer): TokenizeResult { 6 | const deserializer = deserializerFromBuffer(buffer); 7 | return readTokenizeResult(deserializer); 8 | } 9 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/client/types/MaterializedRunResult.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBData } from '@duckdb/data-reader'; 2 | 3 | export interface MaterializedRunResult { 4 | /** 5 | * Full result set. 6 | * 7 | * Includes column metadata, such as types. Supports duplicate column names without renaming. 8 | * 9 | * See the `DuckDBData` interface for details. 10 | */ 11 | data: DuckDBData; 12 | startTimeMs: number; 13 | endTimeMs: number; 14 | } 15 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/util/functions/toBase64.ts: -------------------------------------------------------------------------------- 1 | const encoder = new TextEncoder(); 2 | 3 | export function toBase64(input: string): string { 4 | const encoded = encoder.encode(input); 5 | // For the reason behind this step, see https://developer.mozilla.org/en-US/docs/Web/API/Window/btoa#unicode_strings 6 | const binaryString = Array.from(encoded, (codePoint) => 7 | String.fromCodePoint(codePoint), 8 | ).join(''); 9 | return btoa(binaryString); 10 | } 11 | -------------------------------------------------------------------------------- /ts/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [], 3 | "references": [ 4 | { "path": "pkgs/duckdb-data-reader/src" }, 5 | { "path": "pkgs/duckdb-data-reader/test" }, 6 | { "path": "pkgs/duckdb-data-types/src" }, 7 | { "path": "pkgs/duckdb-data-types/test" }, 8 | { "path": "pkgs/duckdb-data-values/src" }, 9 | { "path": "pkgs/duckdb-data-values/test" }, 10 | { "path": "pkgs/duckdb-ui-client/src" }, 11 | { "path": "pkgs/duckdb-ui-client/test" }, 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/functions/deserializeFromBuffer.ts: -------------------------------------------------------------------------------- 1 | import { BinaryDeserializer } from '../classes/BinaryDeserializer.js'; 2 | import { BinaryStreamReader } from '../classes/BinaryStreamReader.js'; 3 | 4 | export function deserializerFromBuffer( 5 | buffer: ArrayBuffer, 6 | ): BinaryDeserializer { 7 | const streamReader = new BinaryStreamReader(buffer); 8 | const deserializer = new BinaryDeserializer(streamReader); 9 | return deserializer; 10 | } 11 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/types/QueryResult.ts: -------------------------------------------------------------------------------- 1 | import { ColumnNamesAndTypes } from './ColumnNamesAndTypes.js'; 2 | import { DataChunk } from './DataChunk.js'; 3 | 4 | export interface SuccessQueryResult { 5 | success: true; 6 | columnNamesAndTypes: ColumnNamesAndTypes; 7 | chunks: DataChunk[]; 8 | } 9 | 10 | export interface ErrorQueryResult { 11 | success: false; 12 | error: string; 13 | } 14 | 15 | export type QueryResult = SuccessQueryResult | ErrorQueryResult; 16 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # Testing this extension 2 | This directory contains all the tests for this extension. The `sql` directory holds tests that are written as [SQLLogicTests](https://duckdb.org/dev/sqllogictest/intro.html). DuckDB aims to have most its tests in this format as SQL statements, so for the quack extension, this should probably be the goal too. 3 | 4 | The root makefile contains targets to build and run all of these tests. To run the SQLLogicTests: 5 | ```bash 6 | make test 7 | ``` 8 | or 9 | ```bash 10 | make test_debug 11 | ``` -------------------------------------------------------------------------------- /ts/eslint.config.mjs: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | 3 | import eslint from '@eslint/js'; 4 | import tseslint from 'typescript-eslint'; 5 | 6 | export default tseslint.config( 7 | eslint.configs.recommended, 8 | ...tseslint.configs.recommended, 9 | { 10 | rules: { 11 | '@typescript-eslint/no-unused-vars': [ 12 | 'error', 13 | { 14 | argsIgnorePattern: '^_', 15 | varsIgnorePattern: '^_', 16 | caughtErrorsIgnorePattern: '^_', 17 | }, 18 | ], 19 | }, 20 | }, 21 | ); 22 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/util/functions/randomString.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { randomString } from '../../../src/util/functions/randomString'; 3 | 4 | suite('randomString', () => { 5 | test('default length', () => { 6 | expect(randomString().length).toBe(12); 7 | }); 8 | test('custom length', () => { 9 | expect(randomString(5).length).toBe(5); 10 | }); 11 | test('custom chars', () => { 12 | expect(randomString(3, 'xy')).toMatch(/[xy][xy][xy]/); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/conversion/jsonFromDuckDBValue.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBValue } from '../DuckDBValue.js'; 2 | import { Json } from '../Json.js'; 3 | import { SpecialDuckDBValue } from '../SpecialDuckDBValue.js'; 4 | 5 | export function jsonFromDuckDBValue(value: DuckDBValue): Json { 6 | if (value === null) { 7 | return null; 8 | } 9 | if (typeof value === 'bigint') { 10 | return String(value); 11 | } 12 | if (value instanceof SpecialDuckDBValue) { 13 | return value.toJson(); 14 | } 15 | return value; 16 | } 17 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/conversion/hexFromBlob.ts: -------------------------------------------------------------------------------- 1 | export function hexFromBlob( 2 | blob: Uint8Array, 3 | start: number | undefined, 4 | end: number | undefined, 5 | ): string { 6 | if (start === undefined) { 7 | start = 0; 8 | } 9 | if (end === undefined) { 10 | end = blob.length; 11 | } 12 | let hex = ''; 13 | 14 | for (let i = start; i < end; i++) { 15 | const byte = blob[i]; 16 | // Ensure each byte is 2 hex characters 17 | hex += (byte < 16 ? '0' : '') + byte.toString(16); 18 | } 19 | return hex; 20 | } 21 | -------------------------------------------------------------------------------- /ts/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "scripts": { 4 | "build": "tsc -b", 5 | "build:watch": "tsc -b --watch", 6 | "check": "pnpm -r check", 7 | "test": "pnpm -r test" 8 | }, 9 | "devDependencies": { 10 | "typescript": "^5.9.3" 11 | }, 12 | "pnpm": { 13 | "overrides": { 14 | "tar-fs": "^3.0.8", 15 | "ws": "^8.18.1" 16 | } 17 | }, 18 | "packageManager": "pnpm@9.15.2+sha512.93e57b0126f0df74ce6bff29680394c0ba54ec47246b9cf321f0121d8d9bb03f750a705f24edc3c1180853afd7c2c3b94196d0a3d53d3e069d9e2793ef11f321" 19 | } 20 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/conversion/stringFromBlob.ts: -------------------------------------------------------------------------------- 1 | /** Matches BLOB-to-VARCHAR conversion behavior of DuckDB. */ 2 | export function stringFromBlob(bytes: Uint8Array): string { 3 | let result = ''; 4 | for (const byte of bytes) { 5 | if ( 6 | byte <= 0x1f || 7 | byte === 0x22 /* single quote */ || 8 | byte === 0x27 /* double quote */ || 9 | byte >= 0x7f 10 | ) { 11 | result += `\\x${byte.toString(16).toUpperCase().padStart(2, '0')}`; 12 | } else { 13 | result += String.fromCharCode(byte); 14 | } 15 | } 16 | return result; 17 | } 18 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/SpecialDuckDBValue.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBToStringOptions } from './DuckDBToStringOptions.js'; 2 | import { Json } from './Json.js'; 3 | 4 | export abstract class SpecialDuckDBValue { 5 | // The presence of this function can be used to identify SpecialDuckDBValue objects. 6 | public abstract toDuckDBString( 7 | toStringOptions?: DuckDBToStringOptions, 8 | ): string; 9 | 10 | // Convert this value to a SQL-compatible representation. 11 | public abstract toSql(): string; 12 | 13 | public toString(): string { 14 | return this.toDuckDBString(); 15 | } 16 | 17 | public abstract toJson(): Json; 18 | } 19 | -------------------------------------------------------------------------------- /scripts/setup-custom-toolchain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This is an example script that can be used to install additional toolchain dependencies. Feel free to remove this script 4 | # if no additional toolchains are required 5 | 6 | # To enable this script, set the `custom_toolchain_script` option to true when calling the reusable workflow 7 | # `.github/workflows/_extension_distribution.yml` from `https://github.com/duckdb/extension-ci-tools` 8 | 9 | # note that the $DUCKDB_PLATFORM environment variable can be used to discern between the platforms 10 | echo "This is the sample custom toolchain script running for architecture '$DUCKDB_PLATFORM' for the ui extension." 11 | 12 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBBlobValue.ts: -------------------------------------------------------------------------------- 1 | import { stringFromBlob } from './conversion/stringFromBlob.js'; 2 | import { Json } from './Json.js'; 3 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 4 | 5 | export class DuckDBBlobValue extends SpecialDuckDBValue { 6 | public readonly bytes: Uint8Array; 7 | 8 | constructor(bytes: Uint8Array) { 9 | super(); 10 | this.bytes = bytes; 11 | } 12 | 13 | public toDuckDBString(): string { 14 | return stringFromBlob(this.bytes); 15 | } 16 | 17 | public toSql(): string { 18 | return `'${this.toDuckDBString()}'::BLOB`; 19 | } 20 | 21 | public toJson(): Json { 22 | return this.toDuckDBString(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBDateValue.ts: -------------------------------------------------------------------------------- 1 | import { getDuckDBDateStringFromDays } from './conversion/dateTimeStringConversion.js'; 2 | import { Json } from './Json.js'; 3 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 4 | 5 | export class DuckDBDateValue extends SpecialDuckDBValue { 6 | public readonly days: number; 7 | 8 | constructor(days: number) { 9 | super(); 10 | this.days = days; 11 | } 12 | 13 | public toDuckDBString(): string { 14 | return getDuckDBDateStringFromDays(this.days); 15 | } 16 | 17 | public toSql(): string { 18 | return `DATE '${this.toDuckDBString()}'`; 19 | } 20 | 21 | public toJson(): Json { 22 | return this.toDuckDBString(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/include/watcher.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace duckdb { 10 | namespace ui { 11 | struct CatalogState { 12 | std::map db_to_catalog_version; 13 | }; 14 | class HttpServer; 15 | class Watcher { 16 | public: 17 | Watcher(HttpServer &server); 18 | 19 | void Start(); 20 | void Stop(); 21 | 22 | private: 23 | void Watch(); 24 | unique_ptr thread; 25 | std::mutex mutex; 26 | std::condition_variable cv; 27 | std::atomic should_run; 28 | HttpServer &server; 29 | DatabaseInstance *watched_database; 30 | }; 31 | } // namespace ui 32 | } // namespace duckdb 33 | -------------------------------------------------------------------------------- /src/settings.cpp: -------------------------------------------------------------------------------- 1 | #include "settings.hpp" 2 | 3 | #include 4 | 5 | namespace duckdb { 6 | 7 | std::string GetRemoteUrl(const ClientContext &context) { 8 | if (!context.db->config.options.allow_unsigned_extensions) { 9 | return UI_REMOTE_URL_SETTING_DEFAULT; 10 | } 11 | return internal::GetSetting(context, UI_REMOTE_URL_SETTING_NAME); 12 | } 13 | 14 | uint16_t GetLocalPort(const ClientContext &context) { 15 | return internal::GetSetting(context, UI_LOCAL_PORT_SETTING_NAME); 16 | } 17 | 18 | uint32_t GetPollingInterval(const ClientContext &context) { 19 | return internal::GetSetting(context, 20 | UI_POLLING_INTERVAL_SETTING_NAME); 21 | } 22 | } // namespace duckdb 23 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/conversion/duckDBValueToSql.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBValue } from '../DuckDBValue.js'; 2 | import { SpecialDuckDBValue } from '../SpecialDuckDBValue.js'; 3 | 4 | /** Converts a DuckDBValue to a valid SQL string. */ 5 | export function duckDBValueToSql(value: DuckDBValue): string { 6 | if (value == null) { 7 | return 'NULL'; 8 | } 9 | if (typeof value === 'string') { 10 | return `'${value.replace(/'/g, "''")}'`; 11 | } 12 | if (typeof value === 'boolean') { 13 | return value ? 'TRUE' : 'FALSE'; 14 | } 15 | if (typeof value === 'number' || typeof value === 'bigint') { 16 | return String(value); 17 | } 18 | if (value instanceof SpecialDuckDBValue) { 19 | return value.toSql(); 20 | } 21 | return String(value); 22 | } 23 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBTimeValue.ts: -------------------------------------------------------------------------------- 1 | import { getDuckDBTimeStringFromMicrosecondsInDay } from './conversion/dateTimeStringConversion.js'; 2 | import { Json } from './Json.js'; 3 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 4 | 5 | export class DuckDBTimeValue extends SpecialDuckDBValue { 6 | public readonly microseconds: bigint; 7 | 8 | constructor(microseconds: bigint) { 9 | super(); 10 | this.microseconds = microseconds; 11 | } 12 | 13 | public toDuckDBString(): string { 14 | return getDuckDBTimeStringFromMicrosecondsInDay(this.microseconds); 15 | } 16 | 17 | public toSql(): string { 18 | return `TIME '${this.toDuckDBString()}'`; 19 | } 20 | 21 | public toJson(): Json { 22 | return this.toDuckDBString(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBTimestampSecondsValue.ts: -------------------------------------------------------------------------------- 1 | import { getDuckDBTimestampStringFromSeconds } from './conversion/dateTimeStringConversion.js'; 2 | import { Json } from './Json.js'; 3 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 4 | 5 | export class DuckDBTimestampSecondsValue extends SpecialDuckDBValue { 6 | public readonly seconds: bigint; 7 | 8 | constructor(seconds: bigint) { 9 | super(); 10 | this.seconds = seconds; 11 | } 12 | 13 | public toDuckDBString(): string { 14 | return getDuckDBTimestampStringFromSeconds(this.seconds); 15 | } 16 | 17 | public toSql(): string { 18 | return `TIMESTAMP_S '${this.toDuckDBString()}'`; 19 | } 20 | 21 | public toJson(): Json { 22 | return this.toDuckDBString(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/include/state.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace duckdb { 8 | const static std::string STORAGE_EXTENSION_KEY = "ui"; 9 | 10 | class UIStorageExtensionInfo : public StorageExtensionInfo { 11 | public: 12 | static UIStorageExtensionInfo &GetState(const DatabaseInstance &instance); 13 | 14 | shared_ptr FindConnection(const std::string &connection_name); 15 | shared_ptr 16 | FindOrCreateConnection(DatabaseInstance &db, 17 | const std::string &connection_name); 18 | 19 | private: 20 | std::mutex connections_mutex; 21 | std::unordered_map> connections; 22 | }; 23 | 24 | } // namespace duckdb 25 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBTimestampNanosecondsValue.ts: -------------------------------------------------------------------------------- 1 | import { getDuckDBTimestampStringFromNanoseconds } from './conversion/dateTimeStringConversion.js'; 2 | import { Json } from './Json.js'; 3 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 4 | 5 | export class DuckDBTimestampNanosecondsValue extends SpecialDuckDBValue { 6 | public readonly nanoseconds: bigint; 7 | 8 | constructor(nanoseconds: bigint) { 9 | super(); 10 | this.nanoseconds = nanoseconds; 11 | } 12 | 13 | public toDuckDBString(): string { 14 | return getDuckDBTimestampStringFromNanoseconds(this.nanoseconds); 15 | } 16 | 17 | public toSql(): string { 18 | return `TIMESTAMP_NS '${this.toDuckDBString()}'`; 19 | } 20 | 21 | public toJson(): Json { 22 | return this.toDuckDBString(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBTimestampMillisecondsValue.ts: -------------------------------------------------------------------------------- 1 | import { getDuckDBTimestampStringFromMilliseconds } from './conversion/dateTimeStringConversion.js'; 2 | import { Json } from './Json.js'; 3 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 4 | 5 | export class DuckDBTimestampMillisecondsValue extends SpecialDuckDBValue { 6 | public readonly milliseconds: bigint; 7 | 8 | constructor(milliseconds: bigint) { 9 | super(); 10 | this.milliseconds = milliseconds; 11 | } 12 | 13 | public toDuckDBString(): string { 14 | return getDuckDBTimestampStringFromMilliseconds(this.milliseconds); 15 | } 16 | 17 | public toSql(): string { 18 | return `TIMESTAMP_MS '${this.toDuckDBString()}'`; 19 | } 20 | 21 | public toJson(): Json { 22 | return this.toDuckDBString(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/include/event_dispatcher.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace duckdb_httplib_openssl { 9 | class DataSink; 10 | } 11 | 12 | namespace duckdb { 13 | 14 | namespace ui { 15 | 16 | class EventDispatcher { 17 | public: 18 | void SendConnectedEvent(const std::string &token); 19 | void SendCatalogChangedEvent(); 20 | 21 | bool WaitEvent(duckdb_httplib_openssl::DataSink *sink); 22 | void Close(); 23 | 24 | private: 25 | void SendEvent(const std::string &message); 26 | std::mutex mutex; 27 | std::condition_variable cv; 28 | std::atomic_int next_id{0}; 29 | std::atomic_int current_id{-1}; 30 | std::atomic_int wait_count{0}; 31 | std::string message; 32 | std::atomic_bool closed{false}; 33 | }; 34 | } // namespace ui 35 | } // namespace duckdb 36 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBTimestampMicrosecondsValue.ts: -------------------------------------------------------------------------------- 1 | import { getDuckDBTimestampStringFromMicroseconds } from './conversion/dateTimeStringConversion.js'; 2 | import { Json } from './Json.js'; 3 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 4 | 5 | export class DuckDBTimestampMicrosecondsValue extends SpecialDuckDBValue { 6 | public readonly microseconds: bigint; 7 | 8 | constructor(microseconds: bigint) { 9 | super(); 10 | this.microseconds = microseconds; 11 | } 12 | 13 | public toDuckDBString(): string { 14 | return getDuckDBTimestampStringFromMicroseconds(this.microseconds); 15 | } 16 | 17 | public toSql(): string { 18 | return `TIMESTAMP '${this.toDuckDBString()}'`; 19 | } 20 | 21 | public toJson(): Json { 22 | return this.toDuckDBString(); 23 | } 24 | } 25 | 26 | export type DuckDBTimestamp = DuckDBTimestampMicrosecondsValue; 27 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-types/src/DuckDBTypeId.ts: -------------------------------------------------------------------------------- 1 | // copy of DUCKDB_TYPE from the C API, with names shortened 2 | export enum DuckDBTypeId { 3 | INVALID = 0, 4 | BOOLEAN = 1, 5 | TINYINT = 2, 6 | SMALLINT = 3, 7 | INTEGER = 4, 8 | BIGINT = 5, 9 | UTINYINT = 6, 10 | USMALLINT = 7, 11 | UINTEGER = 8, 12 | UBIGINT = 9, 13 | FLOAT = 10, 14 | DOUBLE = 11, 15 | TIMESTAMP = 12, 16 | DATE = 13, 17 | TIME = 14, 18 | INTERVAL = 15, 19 | HUGEINT = 16, 20 | UHUGEINT = 32, 21 | VARCHAR = 17, 22 | BLOB = 18, 23 | DECIMAL = 19, 24 | TIMESTAMP_S = 20, 25 | TIMESTAMP_MS = 21, 26 | TIMESTAMP_NS = 22, 27 | ENUM = 23, 28 | LIST = 24, 29 | STRUCT = 25, 30 | MAP = 26, 31 | ARRAY = 33, 32 | UUID = 27, 33 | UNION = 28, 34 | BIT = 29, 35 | TIME_TZ = 30, 36 | TIMESTAMP_TZ = 31, 37 | ANY = 34, 38 | BIGNUM = 35, 39 | SQLNULL = 36, 40 | STRING_LITERAL = 37, 41 | INTEGER_LITERAL = 38, 42 | } 43 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/constants/LogicalTypeId.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copy of DuckDB's LogicalTypeId. 3 | * 4 | * See LogicalTypeId in https://github.com/duckdb/duckdb/blob/main/src/include/duckdb/common/types.hpp 5 | */ 6 | export const LogicalTypeId = { 7 | BOOLEAN: 10, 8 | TINYINT: 11, 9 | SMALLINT: 12, 10 | INTEGER: 13, 11 | BIGINT: 14, 12 | DATE: 15, 13 | TIME: 16, 14 | TIMESTAMP_SEC: 17, 15 | TIMESTAMP_MS: 18, 16 | TIMESTAMP: 19, 17 | TIMESTAMP_NS: 20, 18 | DECIMAL: 21, 19 | FLOAT: 22, 20 | DOUBLE: 23, 21 | CHAR: 24, 22 | VARCHAR: 25, 23 | BLOB: 26, 24 | INTERVAL: 27, 25 | UTINYINT: 28, 26 | USMALLINT: 29, 27 | UINTEGER: 30, 28 | UBIGINT: 31, 29 | TIMESTAMP_TZ: 32, 30 | TIME_TZ: 34, 31 | BIT: 36, 32 | BIGNUM: 39, 33 | UHUGEINT: 49, 34 | HUGEINT: 50, 35 | UUID: 54, 36 | STRUCT: 100, 37 | LIST: 101, 38 | MAP: 102, 39 | ENUM: 104, 40 | UNION: 107, 41 | ARRAY: 108, 42 | }; 43 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-reader/src/MemoryDuckDBData.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBType } from '@duckdb/data-types'; 2 | import { DuckDBValue } from '@duckdb/data-values'; 3 | import { DuckDBData } from './DuckDBData.js'; 4 | 5 | export class MemoryDuckDBData extends DuckDBData { 6 | constructor( 7 | private columns: { name: string; type: DuckDBType }[], 8 | private values: DuckDBValue[][], 9 | ) { 10 | super(); 11 | } 12 | 13 | get columnCount() { 14 | return this.columns.length; 15 | } 16 | 17 | get rowCount() { 18 | return this.values.length > 0 ? this.values[0].length : 0; 19 | } 20 | 21 | columnName(columnIndex: number): string { 22 | return this.columns[columnIndex].name; 23 | } 24 | 25 | columnType(columnIndex: number): DuckDBType { 26 | return this.columns[columnIndex].type; 27 | } 28 | 29 | value(columnIndex: number, rowIndex: number): DuckDBValue { 30 | return this.values[columnIndex][rowIndex]; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBIntervalValue.ts: -------------------------------------------------------------------------------- 1 | import { getDuckDBIntervalString } from './conversion/dateTimeStringConversion.js'; 2 | import { Json } from './Json.js'; 3 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 4 | 5 | export class DuckDBIntervalValue extends SpecialDuckDBValue { 6 | public readonly months: number; 7 | 8 | public readonly days: number; 9 | 10 | public readonly microseconds: bigint; 11 | 12 | constructor(months: number, days: number, microseconds: bigint) { 13 | super(); 14 | this.months = months; 15 | this.days = days; 16 | this.microseconds = microseconds; 17 | } 18 | 19 | public toDuckDBString(): string { 20 | return getDuckDBIntervalString(this.months, this.days, this.microseconds); 21 | } 22 | 23 | public toSql(): string { 24 | return `INTERVAL '${this.toDuckDBString()}'`; 25 | } 26 | 27 | public toJson(): Json { 28 | return this.toDuckDBString(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/conversion/jsonFromDuckDBValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBListValue } from '../../src'; 3 | import { jsonFromDuckDBValue } from '../../src/conversion/jsonFromDuckDBValue'; 4 | 5 | suite('jsonFromDuckDBValue', () => { 6 | test('null', () => { 7 | expect(jsonFromDuckDBValue(null)).toStrictEqual(null); 8 | }); 9 | test('boolean', () => { 10 | expect(jsonFromDuckDBValue(true)).toStrictEqual(true); 11 | }); 12 | test('number', () => { 13 | expect(jsonFromDuckDBValue(42)).toStrictEqual(42); 14 | }); 15 | test('bigint', () => { 16 | expect(jsonFromDuckDBValue(12345n)).toStrictEqual('12345'); 17 | }); 18 | test('string', () => { 19 | expect(jsonFromDuckDBValue('foo')).toStrictEqual('foo'); 20 | }); 21 | test('special', () => { 22 | expect(jsonFromDuckDBValue(new DuckDBListValue([1, 2, 3]))).toStrictEqual([ 23 | 1, 2, 3, 24 | ]); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /src/utils/env.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/env.hpp" 2 | 3 | #include 4 | #include 5 | 6 | namespace duckdb { 7 | 8 | const char *TryGetEnv(const char *name) { 9 | const char *res = std::getenv(name); 10 | if (res) { 11 | return res; 12 | } 13 | return std::getenv(StringUtil::Upper(name).c_str()); 14 | } 15 | 16 | std::string GetEnvOrDefault(const char *name, const char *default_value) { 17 | const char *res = TryGetEnv(name); 18 | if (res) { 19 | return res; 20 | } 21 | return default_value; 22 | } 23 | 24 | uint32_t GetEnvOrDefaultInt(const char *name, uint32_t default_value) { 25 | const char *res = TryGetEnv(name); 26 | if (res) { 27 | return std::atoi(res); 28 | } 29 | return default_value; 30 | } 31 | 32 | bool IsEnvEnabled(const char *name) { 33 | const char *res = TryGetEnv(name); 34 | if (!res) { 35 | return false; 36 | } 37 | 38 | auto lc_res = StringUtil::Lower(res); 39 | return lc_res == "1" || lc_res == "true"; 40 | } 41 | 42 | } // namespace duckdb 43 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBTimestampTZValue.ts: -------------------------------------------------------------------------------- 1 | import { getDuckDBTimestampStringFromMicroseconds } from './conversion/dateTimeStringConversion.js'; 2 | import { DuckDBToStringOptions } from './DuckDBToStringOptions.js'; 3 | import { Json } from './Json.js'; 4 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 5 | 6 | export class DuckDBTimestampTZValue extends SpecialDuckDBValue { 7 | public readonly microseconds: bigint; 8 | 9 | constructor(microseconds: bigint) { 10 | super(); 11 | this.microseconds = microseconds; 12 | } 13 | 14 | public toDuckDBString(toStringOptions?: DuckDBToStringOptions): string { 15 | return getDuckDBTimestampStringFromMicroseconds( 16 | this.microseconds, 17 | toStringOptions?.timeZone || 'UTC', 18 | ); 19 | } 20 | 21 | public toSql(toStringOptions?: DuckDBToStringOptions): string { 22 | return `TIMESTAMPTZ '${this.toDuckDBString(toStringOptions)}'`; 23 | } 24 | 25 | public toJson(): Json { 26 | return this.toDuckDBString(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018-2025 Stichting DuckDB Foundation 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /src/utils/md_helpers.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/md_helpers.hpp" 2 | 3 | #include 4 | #include 5 | 6 | namespace duckdb { 7 | std::string GetMDToken(Connection &connection) { 8 | if (!IsMDConnected(connection)) { 9 | return ""; // UI expects an empty response if MD isn't connected 10 | } 11 | 12 | auto query_res = connection.Query("CALL GET_MD_TOKEN()"); 13 | if (query_res->HasError()) { 14 | query_res->ThrowError(); 15 | return ""; // unreachable 16 | } 17 | 18 | auto chunk = query_res->Fetch(); 19 | return chunk->GetValue(0, 0).GetValue(); 20 | } 21 | 22 | bool IsMDConnected(Connection &con) { 23 | if (!con.context->db->ExtensionIsLoaded("motherduck")) { 24 | return false; 25 | } 26 | 27 | auto query_res = con.Query("CALL MD_IS_CONNECTED()"); 28 | if (query_res->HasError()) { 29 | std::cerr << "Error in IsMDConnected: " << query_res->GetError() 30 | << std::endl; 31 | return false; 32 | } 33 | 34 | auto chunk = query_res->Fetch(); 35 | return chunk->GetValue(0, 0).GetValue(); 36 | } 37 | } // namespace duckdb 38 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBListValue.ts: -------------------------------------------------------------------------------- 1 | import { displayStringForDuckDBValue } from './conversion/displayStringForDuckDBValue.js'; 2 | import { jsonFromDuckDBValue } from './conversion/jsonFromDuckDBValue.js'; 3 | import { duckDBValueToSql } from './conversion/duckDBValueToSql.js'; 4 | import { DuckDBValue } from './DuckDBValue.js'; 5 | import { Json } from './Json.js'; 6 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 7 | 8 | export class DuckDBListValue extends SpecialDuckDBValue { 9 | public readonly values: readonly DuckDBValue[]; 10 | 11 | constructor(values: readonly DuckDBValue[]) { 12 | super(); 13 | this.values = values; 14 | } 15 | 16 | public toDuckDBString(): string { 17 | const valueStrings = this.values.map(displayStringForDuckDBValue); 18 | return `[${valueStrings.join(', ')}]`; 19 | } 20 | 21 | public toSql(): string { 22 | const valueStrings = this.values.map((v) => duckDBValueToSql(v)); 23 | return `[${valueStrings.join(', ')}]`; 24 | } 25 | 26 | public toJson(): Json { 27 | return this.values.map(jsonFromDuckDBValue); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBArrayValue.ts: -------------------------------------------------------------------------------- 1 | import { displayStringForDuckDBValue } from './conversion/displayStringForDuckDBValue.js'; 2 | import { jsonFromDuckDBValue } from './conversion/jsonFromDuckDBValue.js'; 3 | import { duckDBValueToSql } from './conversion/duckDBValueToSql.js'; 4 | import { DuckDBValue } from './DuckDBValue.js'; 5 | import { Json } from './Json.js'; 6 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 7 | 8 | export class DuckDBArrayValue extends SpecialDuckDBValue { 9 | public readonly values: readonly DuckDBValue[]; 10 | 11 | constructor(values: readonly DuckDBValue[]) { 12 | super(); 13 | this.values = values; 14 | } 15 | 16 | public toDuckDBString(): string { 17 | const valueStrings = this.values.map(displayStringForDuckDBValue); 18 | return `[${valueStrings.join(', ')}]`; 19 | } 20 | 21 | public toSql(): string { 22 | const valueStrings = this.values.map((v) => duckDBValueToSql(v)); 23 | return `[${valueStrings.join(', ')}]`; 24 | } 25 | 26 | public toJson(): Json { 27 | return this.values.map(jsonFromDuckDBValue); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/include/settings.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #define UI_LOCAL_PORT_SETTING_NAME "ui_local_port" 7 | #define UI_LOCAL_PORT_SETTING_DEFAULT 4213 8 | #define UI_REMOTE_URL_SETTING_NAME "ui_remote_url" 9 | #define UI_REMOTE_URL_SETTING_DEFAULT "https://ui.duckdb.org" 10 | #define UI_POLLING_INTERVAL_SETTING_NAME "ui_polling_interval" 11 | #define UI_POLLING_INTERVAL_SETTING_DEFAULT 284 12 | 13 | namespace duckdb { 14 | 15 | namespace internal { 16 | 17 | template 18 | T GetSetting(const ClientContext &context, const char *setting_name) { 19 | Value value; 20 | if (!context.TryGetCurrentSetting(setting_name, value)) { 21 | throw Exception(ExceptionType::SETTINGS, 22 | "Setting \"" + std::string(setting_name) + "\" not found"); 23 | } 24 | return value.GetValue(); 25 | } 26 | } // namespace internal 27 | 28 | std::string GetRemoteUrl(const ClientContext &); 29 | uint16_t GetLocalPort(const ClientContext &); 30 | uint32_t GetPollingInterval(const ClientContext &); 31 | 32 | } // namespace duckdb 33 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/serialization/classes/BinaryStreamReader.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { BinaryStreamReader } from '../../../src/serialization/classes/BinaryStreamReader'; 3 | import { makeBuffer } from '../../helpers/makeBuffer'; 4 | 5 | suite('BinaryStreamReader', () => { 6 | test('basic', () => { 7 | const reader = new BinaryStreamReader( 8 | makeBuffer([11, 22, 33, 44, 0x12, 0x34]), 9 | ); 10 | 11 | expect(reader.getOffset()).toBe(0); 12 | expect(reader.peekUint8()).toBe(11); 13 | expect(reader.readUint8()).toBe(11); 14 | 15 | expect(reader.getOffset()).toBe(1); 16 | expect(reader.peekUint8()).toBe(22); 17 | expect(reader.readUint8()).toBe(22); 18 | 19 | expect(reader.getOffset()).toBe(2); 20 | reader.consume(2); 21 | expect(reader.getOffset()).toBe(4); 22 | expect(reader.peekUint16(false)).toBe(0x1234); 23 | expect(reader.peekUint16(true)).toBe(0x3412); 24 | 25 | const dv = reader.readData(2); 26 | expect(dv.byteLength).toBe(2); 27 | expect(dv.getUint8(0)).toBe(0x12); 28 | expect(dv.getUint8(1)).toBe(0x34); 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /third_party/httplib/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 yhirose 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/client/functions/materializedRunResultFromQueueResult.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBDataReader } from '@duckdb/data-reader'; 2 | import { DuckDBDataChunkIterator } from '../../data-chunk/classes/DuckDBDataChunkIterator.js'; 3 | import { DuckDBUIHttpRequestQueueResult } from '../../http/classes/DuckDBUIHttpRequestQueue.js'; 4 | import { deserializerFromBuffer } from '../../serialization/functions/deserializeFromBuffer.js'; 5 | import { readQueryResult } from '../../serialization/functions/resultReaders.js'; 6 | import { MaterializedRunResult } from '../types/MaterializedRunResult.js'; 7 | 8 | export async function materializedRunResultFromQueueResult( 9 | queueResult: DuckDBUIHttpRequestQueueResult, 10 | ): Promise { 11 | const { buffer, startTimeMs, endTimeMs } = queueResult; 12 | const deserializer = deserializerFromBuffer(buffer); 13 | const result = readQueryResult(deserializer); 14 | if (!result.success) { 15 | throw new Error(result.error); 16 | } 17 | const dataReader = new DuckDBDataReader(new DuckDBDataChunkIterator(result)); 18 | await dataReader.readAll(); 19 | return { data: dataReader, startTimeMs, endTimeMs }; 20 | } 21 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@duckdb/data-values", 3 | "version": "0.0.1", 4 | "description": "Utilities for representing DuckDB values", 5 | "type": "module", 6 | "main": "./out/index.js", 7 | "module": "./out/index.js", 8 | "types": "./out/index.d.ts", 9 | "scripts": { 10 | "preinstall": "pnpm build:src", 11 | "build": "tsc -b src test", 12 | "build:src": "tsc -b src", 13 | "build:test": "tsc -b test", 14 | "build:watch": "tsc -b src test --watch", 15 | "check": "pnpm format:check && pnpm lint", 16 | "clean": "rimraf out", 17 | "format:check": "prettier . --ignore-path $(find-up .prettierignore) --check", 18 | "format:write": "prettier . --ignore-path $(find-up .prettierignore) --write", 19 | "lint": "pnpm eslint src test", 20 | "test": "vitest run", 21 | "test:watch": "vitest" 22 | }, 23 | "devDependencies": { 24 | "@eslint/js": "^9.39.1", 25 | "eslint": "^9.39.1", 26 | "find-up-cli": "^6.0.0", 27 | "prettier": "^3.6.2", 28 | "rimraf": "^6.1.2", 29 | "typescript": "^5.9.3", 30 | "typescript-eslint": "^8.48.0", 31 | "vite": "^6.4.1", 32 | "vitest": "^3.2.4" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/include/utils/serialization.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "duckdb.hpp" 4 | 5 | #include 6 | 7 | namespace duckdb { 8 | namespace ui { 9 | 10 | struct EmptyResult { 11 | void Serialize(duckdb::Serializer &serializer) const; 12 | }; 13 | 14 | struct TokenizeResult { 15 | duckdb::vector offsets; 16 | duckdb::vector types; 17 | 18 | void Serialize(duckdb::Serializer &serializer) const; 19 | }; 20 | 21 | struct ColumnNamesAndTypes { 22 | duckdb::vector names; 23 | duckdb::vector types; 24 | 25 | void Serialize(duckdb::Serializer &serializer) const; 26 | }; 27 | 28 | struct Chunk { 29 | uint16_t row_count; 30 | duckdb::vector vectors; 31 | 32 | void Serialize(duckdb::Serializer &serializer) const; 33 | }; 34 | 35 | struct SuccessResult { 36 | ColumnNamesAndTypes column_names_and_types; 37 | duckdb::vector chunks; 38 | 39 | void Serialize(duckdb::Serializer &serializer) const; 40 | }; 41 | 42 | struct ErrorResult { 43 | std::string error; 44 | 45 | void Serialize(duckdb::Serializer &serializer) const; 46 | }; 47 | 48 | } // namespace ui 49 | } // namespace duckdb 50 | -------------------------------------------------------------------------------- /src/utils/helpers.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/helpers.hpp" 2 | 3 | namespace duckdb { 4 | namespace internal { 5 | 6 | bool ShouldRun(TableFunctionInput &input) { 7 | auto state = 8 | dynamic_cast(input.global_state.get()); 9 | D_ASSERT(state != nullptr); 10 | if (state->run) { 11 | return false; 12 | } 13 | 14 | state->run = true; 15 | return true; 16 | } 17 | 18 | unique_ptr 19 | SingleStringResultBind(ClientContext &, TableFunctionBindInput &, 20 | vector &out_types, 21 | vector &out_names) { 22 | out_names.emplace_back("result"); 23 | out_types.emplace_back(LogicalType::VARCHAR); 24 | return nullptr; 25 | } 26 | 27 | unique_ptr SingleBoolResultBind(ClientContext &, 28 | TableFunctionBindInput &, 29 | vector &out_types, 30 | vector &out_names) { 31 | out_names.emplace_back("result"); 32 | out_types.emplace_back(LogicalType::BOOLEAN); 33 | return nullptr; 34 | } 35 | 36 | } // namespace internal 37 | } // namespace duckdb 38 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBTimeValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBTimeValue } from '../src/DuckDBTimeValue'; 3 | 4 | suite('DuckDBTimeValue', () => { 5 | test('should render a normal time value to the correct string', () => { 6 | expect(new DuckDBTimeValue(45296000000n).toString()).toStrictEqual( 7 | '12:34:56', 8 | ); 9 | }); 10 | test('should render the max time value to the correct string', () => { 11 | expect(new DuckDBTimeValue(86399999999n).toString()).toStrictEqual( 12 | '23:59:59.999999', 13 | ); 14 | }); 15 | test('should render the min time value to the correct string', () => { 16 | expect(new DuckDBTimeValue(0n).toString()).toStrictEqual('00:00:00'); 17 | }); 18 | 19 | suite('toSql', () => { 20 | test('should render time value to SQL', () => { 21 | const micros = BigInt(12 * 3600 + 30 * 60 + 45) * 1000000n + 123456n; 22 | expect(new DuckDBTimeValue(micros).toSql()).toStrictEqual( 23 | "TIME '12:30:45.123456'", 24 | ); 25 | }); 26 | 27 | test('should render midnight to SQL', () => { 28 | expect(new DuckDBTimeValue(0n).toSql()).toStrictEqual("TIME '00:00:00'"); 29 | }); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/types/Vector.ts: -------------------------------------------------------------------------------- 1 | export interface ListEntry { 2 | offset: number; 3 | length: number; 4 | } 5 | 6 | export interface BaseVector { 7 | allValid: number; 8 | validity: DataView | null; 9 | } 10 | 11 | export interface DataVector extends BaseVector { 12 | kind: 'data'; 13 | data: DataView; 14 | } 15 | 16 | export interface StringVector extends BaseVector { 17 | kind: 'string'; 18 | data: string[]; 19 | } 20 | 21 | export interface DataListVector extends BaseVector { 22 | kind: 'datalist'; 23 | data: DataView[]; 24 | } 25 | 26 | export interface VectorListVector extends BaseVector { 27 | kind: 'vectorlist'; 28 | data: Vector[]; 29 | } 30 | 31 | export interface ListVector extends BaseVector { 32 | kind: 'list'; 33 | listSize: number; 34 | entries: ListEntry[]; 35 | child: Vector; 36 | } 37 | 38 | export interface ArrayVector extends BaseVector { 39 | kind: 'array'; 40 | arraySize: number; 41 | child: Vector; 42 | } 43 | 44 | /** See https://github.com/duckdb/duckdb/blob/main/src/include/duckdb/common/types/vector.hpp */ 45 | export type Vector = 46 | | DataVector 47 | | StringVector 48 | | DataListVector 49 | | VectorListVector 50 | | ListVector 51 | | ArrayVector; 52 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/classes/BinaryStreamReader.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Enables reading or peeking at values of a binary buffer. 3 | * Subsequent reads start from the end of the previous one. 4 | */ 5 | export class BinaryStreamReader { 6 | private dv: DataView; 7 | 8 | private offset: number; 9 | 10 | public constructor(buffer: ArrayBuffer) { 11 | this.dv = new DataView(buffer); 12 | this.offset = 0; 13 | } 14 | 15 | public getOffset() { 16 | return this.offset; 17 | } 18 | 19 | public peekUint8() { 20 | return this.dv.getUint8(this.offset); 21 | } 22 | 23 | public peekUint16(le: boolean) { 24 | return this.dv.getUint16(this.offset, le); 25 | } 26 | 27 | public consume(byteCount: number) { 28 | this.offset += byteCount; 29 | } 30 | 31 | private offsetBeforeConsume(byteCount: number) { 32 | const offsetBefore = this.offset; 33 | this.consume(byteCount); 34 | return offsetBefore; 35 | } 36 | 37 | public readUint8() { 38 | return this.dv.getUint8(this.offsetBeforeConsume(1)); 39 | } 40 | 41 | public readData(length: number) { 42 | return new DataView( 43 | this.dv.buffer, 44 | this.offsetBeforeConsume(length), 45 | length, 46 | ); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-types/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@duckdb/data-types", 3 | "version": "0.0.1", 4 | "description": "Utilities for representing DuckDB types", 5 | "type": "module", 6 | "main": "./out/index.js", 7 | "module": "./out/index.js", 8 | "types": "./out/index.d.ts", 9 | "scripts": { 10 | "preinstall": "pnpm build:src", 11 | "build": "tsc -b src test", 12 | "build:src": "tsc -b src", 13 | "build:test": "tsc -b test", 14 | "build:watch": "tsc -b src test --watch", 15 | "check": "pnpm format:check && pnpm lint", 16 | "clean": "rimraf out", 17 | "format:check": "prettier . --ignore-path $(find-up .prettierignore) --check", 18 | "format:write": "prettier . --ignore-path $(find-up .prettierignore) --write", 19 | "lint": "pnpm eslint src test", 20 | "test": "vitest run", 21 | "test:watch": "vitest" 22 | }, 23 | "dependencies": { 24 | "@duckdb/data-values": "workspace:*" 25 | }, 26 | "devDependencies": { 27 | "@eslint/js": "^9.39.1", 28 | "eslint": "^9.39.1", 29 | "find-up-cli": "^6.0.0", 30 | "prettier": "^3.6.2", 31 | "rimraf": "^6.1.2", 32 | "typescript": "^5.9.3", 33 | "typescript-eslint": "^8.48.0", 34 | "vite": "^6.4.1", 35 | "vitest": "^3.2.4" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-reader/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@duckdb/data-reader", 3 | "version": "0.0.1", 4 | "description": "Utilities for representing and reading tabular data returned by DuckDB", 5 | "type": "module", 6 | "main": "./out/index.js", 7 | "module": "./out/index.js", 8 | "types": "./out/index.d.ts", 9 | "scripts": { 10 | "preinstall": "pnpm build:src", 11 | "build": "tsc -b src test", 12 | "build:src": "tsc -b src", 13 | "build:test": "tsc -b test", 14 | "build:watch": "tsc -b src test --watch", 15 | "check": "pnpm format:check && pnpm lint", 16 | "clean": "rimraf out", 17 | "format:check": "prettier . --ignore-path $(find-up .prettierignore) --check", 18 | "format:write": "prettier . --ignore-path $(find-up .prettierignore) --write", 19 | "lint": "pnpm eslint src test", 20 | "test": "vitest run", 21 | "test:watch": "vitest" 22 | }, 23 | "dependencies": { 24 | "@duckdb/data-types": "workspace:*", 25 | "@duckdb/data-values": "workspace:*" 26 | }, 27 | "devDependencies": { 28 | "@eslint/js": "^9.39.1", 29 | "eslint": "^9.39.1", 30 | "find-up-cli": "^6.0.0", 31 | "prettier": "^3.6.2", 32 | "rimraf": "^6.1.2", 33 | "typescript": "^5.9.3", 34 | "typescript-eslint": "^8.48.0", 35 | "vite": "^6.4.1", 36 | "vitest": "^3.2.4" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/types/TypeInfo.ts: -------------------------------------------------------------------------------- 1 | export interface BaseTypeInfo { 2 | alias?: string; 3 | modifiers?: unknown[]; // TODO 4 | } 5 | 6 | export interface GenericTypeInfo extends BaseTypeInfo { 7 | kind: 'generic'; 8 | } 9 | 10 | export interface DecimalTypeInfo extends BaseTypeInfo { 11 | kind: 'decimal'; 12 | width: number; 13 | scale: number; 14 | } 15 | 16 | export interface ListTypeInfo extends BaseTypeInfo { 17 | kind: 'list'; 18 | childType: TypeIdAndInfo; 19 | } 20 | 21 | export interface StructTypeInfo extends BaseTypeInfo { 22 | kind: 'struct'; 23 | childTypes: [string, TypeIdAndInfo][]; 24 | } 25 | 26 | export interface EnumTypeInfo extends BaseTypeInfo { 27 | kind: 'enum'; 28 | valuesCount: number; 29 | values: string[]; 30 | } 31 | 32 | export interface ArrayTypeInfo extends BaseTypeInfo { 33 | kind: 'array'; 34 | childType: TypeIdAndInfo; 35 | size: number; 36 | } 37 | 38 | /** See https://github.com/duckdb/duckdb/blob/main/src/include/duckdb/common/extra_type_info.hpp */ 39 | export type TypeInfo = 40 | | GenericTypeInfo 41 | | DecimalTypeInfo 42 | | ListTypeInfo 43 | | StructTypeInfo 44 | | EnumTypeInfo 45 | | ArrayTypeInfo; 46 | 47 | export interface TypeIdAndInfo { 48 | /** LogicalTypeId */ 49 | id: number; 50 | 51 | /** Extra info for some types. */ 52 | typeInfo?: TypeInfo; 53 | } 54 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@duckdb/ui-client", 3 | "version": "0.0.1", 4 | "description": "Client for communicating with the DuckDB UI server", 5 | "type": "module", 6 | "main": "./out/index.js", 7 | "module": "./out/index.js", 8 | "types": "./out/index.d.ts", 9 | "scripts": { 10 | "preinstall": "pnpm build:src", 11 | "build": "tsc -b src test", 12 | "build:src": "tsc -b src", 13 | "build:test": "tsc -b test", 14 | "build:watch": "tsc -b src test --watch", 15 | "check": "pnpm format:check && pnpm lint", 16 | "clean": "rimraf out", 17 | "format:check": "prettier . --ignore-path $(find-up .prettierignore) --check", 18 | "format:write": "prettier . --ignore-path $(find-up .prettierignore) --write", 19 | "lint": "pnpm eslint src test", 20 | "test": "vitest run", 21 | "test:watch": "vitest" 22 | }, 23 | "dependencies": { 24 | "@duckdb/data-reader": "workspace:*", 25 | "@duckdb/data-types": "workspace:*", 26 | "@duckdb/data-values": "workspace:*", 27 | "core-js": "^3.47.0" 28 | }, 29 | "devDependencies": { 30 | "@eslint/js": "^9.39.1", 31 | "eslint": "^9.39.1", 32 | "find-up-cli": "^6.0.0", 33 | "msw": "^2.12.3", 34 | "prettier": "^3.6.2", 35 | "rimraf": "^6.1.2", 36 | "typescript": "^5.9.3", 37 | "typescript-eslint": "^8.48.0", 38 | "vite": "^6.4.1", 39 | "vitest": "^3.2.4" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBDateValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBDateValue } from '../src/DuckDBDateValue'; 3 | 4 | suite('DuckDBDateValue', () => { 5 | test('should render a normal date value to the correct string', () => { 6 | expect(new DuckDBDateValue(19643).toString()).toStrictEqual('2023-10-13'); 7 | }); 8 | test('should render the max date value to the correct string', () => { 9 | expect(new DuckDBDateValue(2 ** 31 - 2).toString()).toStrictEqual( 10 | '5881580-07-10', 11 | ); 12 | }); 13 | test('should render the min date value to the correct string', () => { 14 | expect(new DuckDBDateValue(-(2 ** 31) + 2).toString()).toStrictEqual( 15 | '5877642-06-25 (BC)', 16 | ); 17 | }); 18 | 19 | suite('toSql', () => { 20 | test('should render a normal date value to SQL format', () => { 21 | expect(new DuckDBDateValue(19643).toSql()).toStrictEqual( 22 | "DATE '2023-10-13'", 23 | ); 24 | }); 25 | test('should render the max date value to SQL format', () => { 26 | expect(new DuckDBDateValue(2 ** 31 - 2).toSql()).toStrictEqual( 27 | "DATE '5881580-07-10'", 28 | ); 29 | }); 30 | test('should render the min date value to SQL format', () => { 31 | expect(new DuckDBDateValue(-(2 ** 31) + 2).toSql()).toStrictEqual( 32 | "DATE '5877642-06-25 (BC)'", 33 | ); 34 | }); 35 | }); 36 | }); 37 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBDecimalValue.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DuckDBDecimalFormatOptions, 3 | stringFromDecimal, 4 | } from './conversion/stringFromDecimal.js'; 5 | import { Json } from './Json.js'; 6 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 7 | 8 | export class DuckDBDecimalValue extends SpecialDuckDBValue { 9 | public readonly scaledValue: bigint; 10 | 11 | public readonly scale: number; 12 | 13 | constructor(scaledValue: bigint, scale: number) { 14 | super(); 15 | this.scaledValue = scaledValue; 16 | this.scale = scale; 17 | } 18 | 19 | public toDuckDBString(): string { 20 | return stringFromDecimal(this.scaledValue, this.scale); 21 | } 22 | 23 | public toSql(): string { 24 | const str = this.toDuckDBString(); 25 | // Calculate width (precision) - total number of digits 26 | const width = ( 27 | this.scaledValue < 0 ? -this.scaledValue : this.scaledValue 28 | ).toString().length; 29 | return `${str}::DECIMAL(${width}, ${this.scale})`; 30 | } 31 | 32 | /** Returns a string representation appropriate to the host environment's current locale. */ 33 | 34 | public toLocaleString( 35 | locales?: string | string[], 36 | options?: DuckDBDecimalFormatOptions, 37 | ): string { 38 | return stringFromDecimal(this.scaledValue, this.scale, { 39 | locales, 40 | options, 41 | }); 42 | } 43 | 44 | public toJson(): Json { 45 | return this.toDuckDBString(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /.github/workflows/TypeScriptWorkspace.yml: -------------------------------------------------------------------------------- 1 | name: TypeScript Workspace 2 | on: 3 | pull_request: 4 | paths: 5 | - "ts/**" 6 | - ".github/workflows/TypeScriptWorkspace.yml" 7 | push: 8 | branches: 9 | - "main" 10 | paths: 11 | - "ts/**" 12 | - ".github/workflows/TypeScriptWorkspace.yml" 13 | workflow_dispatch: 14 | 15 | jobs: 16 | build_and_test: 17 | name: Build & Test 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | 23 | - name: Setup pnpm 24 | uses: pnpm/action-setup@v4 25 | with: 26 | package_json_file: ts/package.json 27 | 28 | - name: Setup Node with pnpm cache 29 | uses: actions/setup-node@v4 30 | with: 31 | cache: 'pnpm' 32 | cache-dependency-path: ts/pnpm-lock.yaml 33 | 34 | # Src files are built using preinstall 35 | - name: Install dependencies & build src 36 | working-directory: ts 37 | run: pnpm install 38 | 39 | # This step is needed to type-check test files. (Src files are built during install.) 40 | - name: Build src & test (to type-check test) 41 | working-directory: ts 42 | run: pnpm build 43 | 44 | - name: Check formatting & linting rules 45 | working-directory: ts 46 | run: pnpm check 47 | 48 | - name: Test 49 | working-directory: ts 50 | run: pnpm test 51 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/data-chunk/classes/DuckDBDataChunk.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBData } from '@duckdb/data-reader'; 2 | import { DuckDBType } from '@duckdb/data-types'; 3 | import { DuckDBValue } from '@duckdb/data-values'; 4 | import { duckDBTypeFromTypeIdAndInfo } from '../../conversion/functions/duckDBTypeFromTypeIdAndInfo.js'; 5 | import { duckDBValueFromVector } from '../../conversion/functions/duckDBValueFromVector.js'; 6 | import { ColumnNamesAndTypes } from '../../serialization/types/ColumnNamesAndTypes.js'; 7 | import { DataChunk } from '../../serialization/types/DataChunk.js'; 8 | 9 | export class DuckDBDataChunk extends DuckDBData { 10 | constructor( 11 | private columnNamesAndTypes: ColumnNamesAndTypes, 12 | private chunk: DataChunk, 13 | ) { 14 | super(); 15 | } 16 | 17 | get columnCount() { 18 | return this.columnNamesAndTypes.names.length; 19 | } 20 | 21 | get rowCount() { 22 | return this.chunk.rowCount; 23 | } 24 | 25 | columnName(columnIndex: number): string { 26 | return this.columnNamesAndTypes.names[columnIndex]; 27 | } 28 | 29 | columnType(columnIndex: number): DuckDBType { 30 | return duckDBTypeFromTypeIdAndInfo( 31 | this.columnNamesAndTypes.types[columnIndex], 32 | ); 33 | } 34 | 35 | value(columnIndex: number, rowIndex: number): DuckDBValue { 36 | return duckDBValueFromVector( 37 | this.columnNamesAndTypes.types[columnIndex], 38 | this.chunk.vectors[columnIndex], 39 | rowIndex, 40 | ); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBTimeTZValue.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getDuckDBTimeStringFromMicrosecondsInDay, 3 | getOffsetStringFromSeconds, 4 | } from './conversion/dateTimeStringConversion.js'; 5 | import { Json } from './Json.js'; 6 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 7 | 8 | export class DuckDBTimeTZValue extends SpecialDuckDBValue { 9 | public readonly micros: bigint; 10 | public readonly offset: number; 11 | 12 | constructor(micros: bigint, offset: number) { 13 | super(); 14 | this.micros = micros; 15 | this.offset = offset; 16 | } 17 | 18 | public toDuckDBString(): string { 19 | return `${getDuckDBTimeStringFromMicrosecondsInDay( 20 | this.micros, 21 | )}${getOffsetStringFromSeconds(this.offset)}`; 22 | } 23 | 24 | public toSql(): string { 25 | return `TIMETZ '${this.toDuckDBString()}'`; 26 | } 27 | 28 | public toJson(): Json { 29 | return this.toDuckDBString(); 30 | } 31 | 32 | private static TimeBits = 40; 33 | private static OffsetBits = 24; 34 | private static MaxOffset = 16 * 60 * 60 - 1; // ±15:59:59 = 57599 seconds 35 | 36 | public static fromBits(bits: bigint): DuckDBTimeTZValue { 37 | const micros = BigInt.asUintN( 38 | DuckDBTimeTZValue.TimeBits, 39 | bits >> BigInt(DuckDBTimeTZValue.OffsetBits), 40 | ); 41 | const offset = 42 | DuckDBTimeTZValue.MaxOffset - 43 | Number(BigInt.asUintN(DuckDBTimeTZValue.OffsetBits, bits)); 44 | return new DuckDBTimeTZValue(micros, offset); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBMapValue.ts: -------------------------------------------------------------------------------- 1 | import { displayStringForDuckDBValue } from './conversion/displayStringForDuckDBValue.js'; 2 | import { jsonFromDuckDBValue } from './conversion/jsonFromDuckDBValue.js'; 3 | import { duckDBValueToSql } from './conversion/duckDBValueToSql.js'; 4 | import { DuckDBMapEntry } from './DuckDBMapEntry.js'; 5 | import { Json } from './Json.js'; 6 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 7 | 8 | export class DuckDBMapValue extends SpecialDuckDBValue { 9 | public readonly entries: readonly DuckDBMapEntry[]; 10 | 11 | constructor(entries: readonly DuckDBMapEntry[]) { 12 | super(); 13 | this.entries = entries; 14 | } 15 | 16 | public toDuckDBString(): string { 17 | const entryStrings = this.entries.map( 18 | ({ key, value }) => 19 | `${displayStringForDuckDBValue(key)}: ${displayStringForDuckDBValue( 20 | value, 21 | )}`, 22 | ); 23 | return `{${entryStrings.join(', ')}}`; 24 | } 25 | 26 | public toSql(): string { 27 | const entryStrings = this.entries.map( 28 | ({ key, value }) => 29 | `${duckDBValueToSql(key)}: ${duckDBValueToSql(value)}`, 30 | ); 31 | return `MAP {${entryStrings.join(', ')}}`; 32 | } 33 | 34 | public toJson(): Json { 35 | const result: Json = {}; 36 | for (const { key, value } of this.entries) { 37 | const keyString = displayStringForDuckDBValue(key); 38 | result[keyString] = jsonFromDuckDBValue(value); 39 | } 40 | return result; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/conversion/functions/vectorGetters.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ArrayVector, 3 | DataListVector, 4 | DataVector, 5 | ListVector, 6 | StringVector, 7 | Vector, 8 | VectorListVector, 9 | } from '../../serialization/types/Vector.js'; 10 | 11 | export function getDataVector(vector: Vector): DataVector { 12 | if (vector.kind !== 'data') { 13 | throw new Error(`Unexpected vector.kind: ${vector.kind}`); 14 | } 15 | return vector; 16 | } 17 | 18 | export function getStringVector(vector: Vector): StringVector { 19 | if (vector.kind !== 'string') { 20 | throw new Error(`Unexpected vector.kind: ${vector.kind}`); 21 | } 22 | return vector; 23 | } 24 | 25 | export function getDataListVector(vector: Vector): DataListVector { 26 | if (vector.kind !== 'datalist') { 27 | throw new Error(`Unexpected vector.kind: ${vector.kind}`); 28 | } 29 | return vector; 30 | } 31 | 32 | export function getVectorListVector(vector: Vector): VectorListVector { 33 | if (vector.kind !== 'vectorlist') { 34 | throw new Error(`Unexpected vector.kind: ${vector.kind}`); 35 | } 36 | return vector; 37 | } 38 | 39 | export function getListVector(vector: Vector): ListVector { 40 | if (vector.kind !== 'list') { 41 | throw new Error(`Unexpected vector.kind: ${vector.kind}`); 42 | } 43 | return vector; 44 | } 45 | 46 | export function getArrayVector(vector: Vector): ArrayVector { 47 | if (vector.kind !== 'array') { 48 | throw new Error(`Unexpected vector.kind: ${vector.kind}`); 49 | } 50 | return vector; 51 | } 52 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/http/functions/makeDuckDBUIHttpRequestHeaders.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { makeDuckDBUIHttpRequestHeaders } from '../../../src/http/functions/makeDuckDBUIHttpRequestHeaders'; 3 | 4 | suite('makeDuckDBUIHttpRequestHeaders', () => { 5 | test('description', () => { 6 | expect([ 7 | ...makeDuckDBUIHttpRequestHeaders({ 8 | description: 'example description', 9 | }).entries(), 10 | ]).toEqual([['x-duckdb-ui-request-description', 'example description']]); 11 | }); 12 | test('connection name', () => { 13 | expect([ 14 | ...makeDuckDBUIHttpRequestHeaders({ 15 | connectionName: 'example connection name', 16 | }).entries(), 17 | ]).toEqual([['x-duckdb-ui-connection-name', 'example connection name']]); 18 | }); 19 | test('database name', () => { 20 | // should be base64 encoded 21 | expect([ 22 | ...makeDuckDBUIHttpRequestHeaders({ 23 | databaseName: 'example database name', 24 | }).entries(), 25 | ]).toEqual([['x-duckdb-ui-database-name', 'ZXhhbXBsZSBkYXRhYmFzZSBuYW1l']]); 26 | }); 27 | test('parameters', () => { 28 | // values should be base64 encoded 29 | expect([ 30 | ...makeDuckDBUIHttpRequestHeaders({ 31 | parameters: ['first', 'second'], 32 | }).entries(), 33 | ]).toEqual([ 34 | ['x-duckdb-ui-parameter-count', '2'], 35 | ['x-duckdb-ui-parameter-value-0', 'Zmlyc3Q='], 36 | ['x-duckdb-ui-parameter-value-1', 'c2Vjb25k'], 37 | ]); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/data-chunk/classes/DuckDBDataChunkIterator.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AsyncDuckDBDataBatchIterator, 3 | DuckDBData, 4 | DuckDBDataBatchIteratorResult, 5 | } from '@duckdb/data-reader'; 6 | import { SuccessQueryResult } from '../../serialization/types/QueryResult.js'; 7 | import { DuckDBDataChunk } from './DuckDBDataChunk.js'; 8 | 9 | const ITERATOR_DONE: DuckDBDataBatchIteratorResult = Object.freeze({ 10 | done: true, 11 | value: undefined, 12 | }); 13 | 14 | export class DuckDBDataChunkIterator implements AsyncDuckDBDataBatchIterator { 15 | private result: SuccessQueryResult; 16 | 17 | private index: number; 18 | 19 | constructor(result: SuccessQueryResult) { 20 | this.result = result; 21 | this.index = 0; 22 | } 23 | 24 | async next(): Promise { 25 | if (this.index < this.result.chunks.length) { 26 | return { 27 | done: false, 28 | value: new DuckDBDataChunk( 29 | this.result.columnNamesAndTypes, 30 | this.result.chunks[this.index++], 31 | ), 32 | }; 33 | } 34 | return ITERATOR_DONE; 35 | } 36 | 37 | async return(value?: DuckDBData): Promise { 38 | if (value) { 39 | return { done: true, value }; 40 | } 41 | return ITERATOR_DONE; 42 | } 43 | 44 | async throw(_e?: unknown): Promise { 45 | return ITERATOR_DONE; 46 | } 47 | 48 | [Symbol.asyncIterator](): AsyncDuckDBDataBatchIterator { 49 | return this; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-types/src/extensionTypes.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DOUBLE, 3 | DuckDBBlobType, 4 | DuckDBVarCharType, 5 | FLOAT, 6 | HUGEINT, 7 | LIST, 8 | STRUCT, 9 | USMALLINT, 10 | UTINYINT, 11 | } from './DuckDBType.js'; 12 | 13 | // see https://github.com/duckdb/duckdb-inet/blob/main/src/inet_extension.cpp 14 | export const INET = STRUCT( 15 | { ip_type: UTINYINT, address: HUGEINT, mask: USMALLINT }, 16 | 'INET', 17 | ); 18 | 19 | // see LogicalType::JSON() in https://github.com/duckdb/duckdb/blob/main/src/common/types.cpp 20 | export const JSONType = DuckDBVarCharType.create('JSON'); 21 | 22 | // see https://github.com/duckdb/duckdb-spatial/blob/main/src/spatial/spatial_types.cpp 23 | export const BOX_2D = STRUCT( 24 | { min_x: DOUBLE, min_y: DOUBLE, max_x: DOUBLE, max_y: DOUBLE }, 25 | 'BOX_2D', 26 | ); 27 | export const BOX_2DF = STRUCT( 28 | { min_x: FLOAT, min_y: FLOAT, max_x: FLOAT, max_y: FLOAT }, 29 | 'BOX_2DF', 30 | ); 31 | export const GEOMETRY = DuckDBBlobType.create('GEOMETRY'); 32 | export const LINESTRING_2D = LIST( 33 | STRUCT({ x: DOUBLE, y: DOUBLE }), 34 | 'LINESTRING_2D', 35 | ); 36 | export const POINT_2D = STRUCT({ x: DOUBLE, y: DOUBLE }, 'POINT_2D'); 37 | export const POINT_3D = STRUCT({ x: DOUBLE, y: DOUBLE, z: DOUBLE }, 'POINT_3D'); 38 | export const POINT_4D = STRUCT( 39 | { x: DOUBLE, y: DOUBLE, z: DOUBLE, m: DOUBLE }, 40 | 'POINT_4D', 41 | ); 42 | export const POLYGON_2D = LIST( 43 | LIST(STRUCT({ x: DOUBLE, y: DOUBLE })), 44 | 'POLYGON_2D', 45 | ); 46 | export const WKB_BLOB = DuckDBBlobType.create('WKB_BLOB'); 47 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBStructValue.ts: -------------------------------------------------------------------------------- 1 | import { displayStringForDuckDBValue } from './conversion/displayStringForDuckDBValue.js'; 2 | import { jsonFromDuckDBValue } from './conversion/jsonFromDuckDBValue.js'; 3 | import { duckDBValueToSql } from './conversion/duckDBValueToSql.js'; 4 | import { DuckDBStructEntry } from './DuckDBStructEntry.js'; 5 | import { Json } from './Json.js'; 6 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 7 | 8 | export class DuckDBStructValue extends SpecialDuckDBValue { 9 | public readonly entries: readonly DuckDBStructEntry[]; 10 | 11 | constructor(entries: readonly DuckDBStructEntry[]) { 12 | super(); 13 | this.entries = entries; 14 | } 15 | 16 | public toDuckDBString(): string { 17 | const entryStrings = this.entries.map( 18 | ({ key, value }) => 19 | `${displayStringForDuckDBValue(key)}: ${displayStringForDuckDBValue( 20 | value, 21 | )}`, 22 | ); 23 | return `{${entryStrings.join(', ')}}`; 24 | } 25 | 26 | public toSql(): string { 27 | if (this.entries.length === 0) { 28 | throw new Error('Empty structs cannot be represented as SQL literals'); 29 | } 30 | const entryStrings = this.entries.map( 31 | ({ key, value }) => 32 | `${duckDBValueToSql(key)}: ${duckDBValueToSql(value)}`, 33 | ); 34 | return `{${entryStrings.join(', ')}}`; 35 | } 36 | 37 | public toJson(): Json { 38 | const result: Json = {}; 39 | for (const { key, value } of this.entries) { 40 | const keyString = displayStringForDuckDBValue(key); 41 | result[keyString] = jsonFromDuckDBValue(value); 42 | } 43 | return result; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/conversion/getBigNumFromBytes.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Returns the JS bigint value represented by the byte array of a BIGNUM in DuckDB's internal format. 3 | * 4 | * DuckDB stores BIGNUMs as an array of bytes consisting of a three-byte header followed by a variable number of bytes 5 | * (at least one). The header specifies the number of bytes after the header, and whether the number is positive or 6 | * negative. The bytes after the header specify the absolute value of the number, in big endian format. 7 | * 8 | * The sign of the number is determined by the MSB of the header, which is 1 for positive and 0 for negative. Negative 9 | * numbers also have all bytes of both the header and value inverted. (For negative numbers, the MSB is 0 after this 10 | * inversion. Put another way: the MSB of the header is always 1, but it's inverted for negative numbers.) 11 | */ 12 | export function getBigNumFromBytes(bytes: Uint8Array): bigint { 13 | const firstByte = bytes[0]; 14 | const positive = (firstByte & 0x80) > 0; 15 | const uint64Mask = positive ? 0n : 0xffffffffffffffffn; 16 | const uint8Mask = positive ? 0 : 0xff; 17 | const dv = new DataView( 18 | bytes.buffer, 19 | bytes.byteOffset + 3, 20 | bytes.byteLength - 3, 21 | ); 22 | const lastUint64Offset = dv.byteLength - 8; 23 | let offset = 0; 24 | let result = 0n; 25 | while (offset <= lastUint64Offset) { 26 | result = (result << 64n) | (dv.getBigUint64(offset) ^ uint64Mask); 27 | offset += 8; 28 | } 29 | while (offset < dv.byteLength) { 30 | result = (result << 8n) | BigInt(dv.getUint8(offset) ^ uint8Mask); 31 | offset += 1; 32 | } 33 | return positive ? result : -result; 34 | } 35 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/index.ts: -------------------------------------------------------------------------------- 1 | export { duckDBValueToSql } from './conversion/duckDBValueToSql.js'; 2 | export { getBigNumFromBytes } from './conversion/getBigNumFromBytes.js'; 3 | export { jsonFromDuckDBValue } from './conversion/jsonFromDuckDBValue.js'; 4 | export { DuckDBArrayValue } from './DuckDBArrayValue.js'; 5 | export { DuckDBBitValue } from './DuckDBBitValue.js'; 6 | export { DuckDBBlobValue } from './DuckDBBlobValue.js'; 7 | export { DuckDBDateValue } from './DuckDBDateValue.js'; 8 | export { DuckDBDecimalValue } from './DuckDBDecimalValue.js'; 9 | export { DuckDBIntervalValue } from './DuckDBIntervalValue.js'; 10 | export { DuckDBListValue } from './DuckDBListValue.js'; 11 | export { DuckDBMapEntry } from './DuckDBMapEntry.js'; 12 | export { DuckDBMapValue } from './DuckDBMapValue.js'; 13 | export { DuckDBStructEntry } from './DuckDBStructEntry.js'; 14 | export { DuckDBStructValue } from './DuckDBStructValue.js'; 15 | export { DuckDBTimestampMicrosecondsValue } from './DuckDBTimestampMicrosecondsValue.js'; 16 | export { DuckDBTimestampMillisecondsValue } from './DuckDBTimestampMillisecondsValue.js'; 17 | export { DuckDBTimestampNanosecondsValue } from './DuckDBTimestampNanosecondsValue.js'; 18 | export { DuckDBTimestampSecondsValue } from './DuckDBTimestampSecondsValue.js'; 19 | export { DuckDBTimestampTZValue } from './DuckDBTimestampTZValue.js'; 20 | export { DuckDBTimeTZValue } from './DuckDBTimeTZValue.js'; 21 | export { DuckDBTimeValue } from './DuckDBTimeValue.js'; 22 | export { DuckDBToStringOptions } from './DuckDBToStringOptions.js'; 23 | export { DuckDBUUIDValue } from './DuckDBUUIDValue.js'; 24 | export { DuckDBValue } from './DuckDBValue.js'; 25 | export { Json } from './Json.js'; 26 | export { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 27 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-reader/src/ColumnFilteredDuckDBData.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBType } from '@duckdb/data-types'; 2 | import { DuckDBValue } from '@duckdb/data-values'; 3 | import { DuckDBData } from './DuckDBData.js'; 4 | 5 | export class ColumnFilteredDuckDBData extends DuckDBData { 6 | private readonly inputColumnIndexForOutputColumnIndex: readonly number[]; 7 | 8 | constructor( 9 | private data: DuckDBData, 10 | columnVisibility: readonly boolean[], 11 | ) { 12 | super(); 13 | 14 | const inputColumnIndexForOutputColumnIndex: number[] = []; 15 | const inputColumnCount = data.columnCount; 16 | let inputIndex = 0; 17 | while (inputIndex < inputColumnCount) { 18 | while (inputIndex < inputColumnCount && !columnVisibility[inputIndex]) { 19 | inputIndex++; 20 | } 21 | if (inputIndex < inputColumnCount) { 22 | inputColumnIndexForOutputColumnIndex.push(inputIndex++); 23 | } 24 | } 25 | this.inputColumnIndexForOutputColumnIndex = 26 | inputColumnIndexForOutputColumnIndex; 27 | } 28 | 29 | get columnCount() { 30 | return this.inputColumnIndexForOutputColumnIndex.length; 31 | } 32 | 33 | get rowCount() { 34 | return this.data.rowCount; 35 | } 36 | 37 | columnName(columnIndex: number): string { 38 | return this.data.columnName( 39 | this.inputColumnIndexForOutputColumnIndex[columnIndex], 40 | ); 41 | } 42 | 43 | columnType(columnIndex: number): DuckDBType { 44 | return this.data.columnType( 45 | this.inputColumnIndexForOutputColumnIndex[columnIndex], 46 | ); 47 | } 48 | 49 | value(columnIndex: number, rowIndex: number): DuckDBValue { 50 | return this.data.value( 51 | this.inputColumnIndexForOutputColumnIndex[columnIndex], 52 | rowIndex, 53 | ); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/functions/basicReaders.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BinaryDeserializer, 3 | ListReader, 4 | Reader, 5 | } from '../classes/BinaryDeserializer.js'; 6 | 7 | export function readUnsupported(deserializer: BinaryDeserializer): void { 8 | deserializer.throwUnsupported(); 9 | } 10 | 11 | export function readNullable( 12 | deserializer: BinaryDeserializer, 13 | reader: Reader, 14 | ): T | null { 15 | return deserializer.readNullable(reader); 16 | } 17 | 18 | export function readUint8(deserializer: BinaryDeserializer): number { 19 | return deserializer.readUint8(); 20 | } 21 | 22 | export function readBoolean(deserializer: BinaryDeserializer): boolean { 23 | return deserializer.readUint8() !== 0; 24 | } 25 | 26 | export function readVarInt(deserializer: BinaryDeserializer): number { 27 | return deserializer.readVarInt(); 28 | } 29 | 30 | export function readVarIntList(deserializer: BinaryDeserializer): number[] { 31 | return readList(deserializer, readVarInt); 32 | } 33 | 34 | export function readData(deserializer: BinaryDeserializer): DataView { 35 | return deserializer.readData(); 36 | } 37 | 38 | export function readDataList(deserializer: BinaryDeserializer): DataView[] { 39 | return readList(deserializer, readData); 40 | } 41 | 42 | export function readString(deserializer: BinaryDeserializer): string { 43 | return deserializer.readString(); 44 | } 45 | 46 | export function readList( 47 | deserializer: BinaryDeserializer, 48 | reader: ListReader, 49 | ): T[] { 50 | return deserializer.readList(reader); 51 | } 52 | 53 | export function readStringList(deserializer: BinaryDeserializer): string[] { 54 | return readList(deserializer, readString); 55 | } 56 | 57 | export function readPair( 58 | deserializer: BinaryDeserializer, 59 | firstReader: Reader, 60 | secondReader: Reader, 61 | ): [T, U] { 62 | return deserializer.readPair(firstReader, secondReader); 63 | } 64 | -------------------------------------------------------------------------------- /docs/UPDATING.md: -------------------------------------------------------------------------------- 1 | # Extension updating 2 | When cloning this template, the target version of DuckDB should be the latest stable release of DuckDB. However, there 3 | will inevitably come a time when a new DuckDB is released and the extension repository needs updating. This process goes 4 | as follows: 5 | 6 | - Bump submodules 7 | - `./duckdb` should be set to latest tagged release 8 | - `./extension-ci-tools` should be set to updated branch corresponding to latest DuckDB release. So if you're building for DuckDB `v1.1.0` there will be a branch in `extension-ci-tools` named `v1.1.0` to which you should check out. 9 | - Bump versions in `./github/workflows` 10 | - `duckdb_version` input in `duckdb-stable-build` job in `MainDistributionPipeline.yml` should be set to latest tagged release 11 | - `duckdb_version` input in `duckdb-stable-deploy` job in `MainDistributionPipeline.yml` should be set to latest tagged release 12 | - the reusable workflow `duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml` for the `duckdb-stable-build` job should be set to latest tagged release 13 | 14 | # API changes 15 | DuckDB extensions built with this extension template are built against the internal C++ API of DuckDB. This API is not guaranteed to be stable. 16 | What this means for extension development is that when updating your extensions DuckDB target version using the above steps, you may run into the fact that your extension no longer builds properly. 17 | 18 | Currently, DuckDB does not (yet) provide a specific change log for these API changes, but it is generally not too hard to figure out what has changed. 19 | 20 | For figuring out how and why the C++ API changed, we recommend using the following resources: 21 | - DuckDB's [Release Notes](https://github.com/duckdb/duckdb/releases) 22 | - DuckDB's history of [Core extension patches](https://github.com/duckdb/duckdb/commits/main/.github/patches/extensions) 23 | - The git history of the relevant C++ Header file of the API that has changed -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/http/functions/sendDuckDBUIHttpRequest.test.ts: -------------------------------------------------------------------------------- 1 | import { http, HttpResponse } from 'msw'; 2 | import { expect, suite, test } from 'vitest'; 3 | import { sendDuckDBUIHttpRequest } from '../../../src/http/functions/sendDuckDBUIHttpRequest'; 4 | import { makeBuffer } from '../../helpers/makeBuffer'; 5 | import { mockRequests } from '../../helpers/mockRequests'; 6 | 7 | suite('sendDuckDBUIHttpRequest', () => { 8 | test('basic', async () => { 9 | return mockRequests( 10 | [ 11 | http.post('http://localhost/example/path', () => { 12 | return HttpResponse.arrayBuffer(makeBuffer([17, 42])); 13 | }), 14 | ], 15 | async () => { 16 | await expect( 17 | sendDuckDBUIHttpRequest( 18 | 'http://localhost/example/path', 19 | 'example body', 20 | ), 21 | ).resolves.toEqual(makeBuffer([17, 42])); 22 | }, 23 | ); 24 | }); 25 | test('headers', async () => { 26 | return mockRequests( 27 | [ 28 | http.post('http://localhost/example/path', ({ request }) => { 29 | if ( 30 | request.headers.get('X-Example-Header-1') !== 31 | 'example-header-1-value' || 32 | request.headers.get('X-Example-Header-2') !== 33 | 'example-header-2-value' 34 | ) { 35 | return HttpResponse.error(); 36 | } 37 | return HttpResponse.arrayBuffer(makeBuffer([17, 42])); 38 | }), 39 | ], 40 | async () => { 41 | const headers = new Headers(); 42 | headers.append('X-Example-Header-1', 'example-header-1-value'); 43 | headers.append('X-Example-Header-2', 'example-header-2-value'); 44 | await expect( 45 | sendDuckDBUIHttpRequest( 46 | 'http://localhost/example/path', 47 | 'example body', 48 | headers, 49 | ), 50 | ).resolves.toEqual(makeBuffer([17, 42])); 51 | }, 52 | ); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/state.cpp: -------------------------------------------------------------------------------- 1 | #include "state.hpp" 2 | 3 | #include 4 | 5 | namespace duckdb { 6 | 7 | UIStorageExtensionInfo & 8 | UIStorageExtensionInfo::GetState(const DatabaseInstance &instance) { 9 | auto &config = instance.config; 10 | auto it = config.storage_extensions.find(STORAGE_EXTENSION_KEY); 11 | if (it == config.storage_extensions.end()) { 12 | throw std::runtime_error( 13 | "Fatal error: couldn't find the UI extension state."); 14 | } 15 | return *static_cast(it->second->storage_info.get()); 16 | } 17 | 18 | shared_ptr 19 | UIStorageExtensionInfo::FindConnection(const std::string &connection_name) { 20 | if (connection_name.empty()) { 21 | return nullptr; 22 | } 23 | 24 | // Need to protect access to the connections map because this can be called 25 | // from multiple threads. 26 | std::lock_guard guard(connections_mutex); 27 | 28 | auto result = connections.find(connection_name); 29 | if (result != connections.end()) { 30 | return result->second; 31 | } 32 | 33 | return nullptr; 34 | } 35 | 36 | shared_ptr UIStorageExtensionInfo::FindOrCreateConnection( 37 | DatabaseInstance &db, const std::string &connection_name) { 38 | if (connection_name.empty()) { 39 | // If no connection name was provided, create and return a new connection 40 | // but don't remember it. 41 | return make_shared_ptr(db); 42 | } 43 | 44 | // If an existing connection with the provided name was found, return it. 45 | auto connection = FindConnection(connection_name); 46 | if (connection) { 47 | return connection; 48 | } 49 | 50 | // Otherwise, create a new one, remember it, and return it. 51 | auto new_con = make_shared_ptr(db); 52 | 53 | // Need to protect access to the connections map because this can be called 54 | // from multiple threads. 55 | std::lock_guard guard(connections_mutex); 56 | connections[connection_name] = new_con; 57 | return new_con; 58 | } 59 | 60 | } // namespace duckdb 61 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBUUIDValue.ts: -------------------------------------------------------------------------------- 1 | import { hexFromBlob } from './conversion/hexFromBlob.js'; 2 | import { Json } from './Json.js'; 3 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 4 | 5 | export class DuckDBUUIDValue extends SpecialDuckDBValue { 6 | public readonly bytes: Uint8Array; 7 | 8 | constructor(bytes: Uint8Array) { 9 | super(); 10 | this.bytes = bytes; 11 | } 12 | 13 | public toDuckDBString(): string { 14 | if (this.bytes.length !== 16) { 15 | throw new Error('Invalid UUID bytes length'); 16 | } 17 | 18 | // Insert dashes to format the UUID 19 | return `${hexFromBlob(this.bytes, 0, 4)}-${hexFromBlob(this.bytes, 4, 6)}-${hexFromBlob(this.bytes, 6, 8)}-${hexFromBlob(this.bytes, 8, 10)}-${hexFromBlob(this.bytes, 10, 16)}`; 20 | } 21 | 22 | public toSql(): string { 23 | return `'${this.toDuckDBString()}'::UUID`; 24 | } 25 | 26 | public toJson(): Json { 27 | return this.toDuckDBString(); 28 | } 29 | 30 | /** 31 | * Create a DuckDBUUIDValue value from a HUGEINT as stored by DuckDB. 32 | * 33 | * UUID values are stored with their MSB flipped so their numeric ordering matches their string ordering. 34 | */ 35 | public static fromStoredHugeint(hugeint: bigint): DuckDBUUIDValue { 36 | // Flip the MSB and truncate to 128 bits to extract the represented unsigned 128-bit value. 37 | const uint128 = 38 | (hugeint ^ 0x80000000000000000000000000000000n) & 39 | 0xffffffffffffffffffffffffffffffffn; 40 | return DuckDBUUIDValue.fromUint128(uint128); 41 | } 42 | 43 | /** Create a DuckDBUUIDValue value from an unsigned 128-bit integer in a JS BigInt. */ 44 | public static fromUint128(uint128: bigint): DuckDBUUIDValue { 45 | const bytes = new Uint8Array(16); 46 | const dv = new DataView(bytes.buffer); 47 | // Write the unsigned 128-bit integer to the buffer in big endian format. 48 | dv.setBigUint64(0, BigInt.asUintN(64, uint128 >> BigInt(64)), false); 49 | dv.setBigUint64(8, BigInt.asUintN(64, uint128), false); 50 | return new DuckDBUUIDValue(bytes); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/utils/encoding.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/encoding.hpp" 2 | 3 | #include 4 | #include 5 | 6 | namespace duckdb { 7 | 8 | // Copied from 9 | // https://www.mycplus.com/source-code/c-source-code/base64-encode-decode/ 10 | constexpr char k_encoding_table[] = 11 | "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+_"; 12 | 13 | std::vector BuildDecodingTable() { 14 | std::vector decoding_table; 15 | decoding_table.resize(256); 16 | for (int i = 0; i < 64; ++i) { 17 | decoding_table[static_cast(k_encoding_table[i])] = i; 18 | } 19 | return decoding_table; 20 | } 21 | 22 | const static std::vector k_decoding_table = BuildDecodingTable(); 23 | 24 | std::string DecodeBase64(const std::string &data) { 25 | size_t input_length = data.size(); 26 | if (input_length < 4 || input_length % 4 != 0) { 27 | // Handle this exception 28 | return ""; 29 | } 30 | 31 | size_t output_length = input_length / 4 * 3; 32 | if (data[input_length - 1] == '=') { 33 | output_length--; 34 | } 35 | if (data[input_length - 2] == '=') { 36 | output_length--; 37 | } 38 | 39 | std::string decoded_data; 40 | decoded_data.resize(output_length); 41 | for (size_t i = 0, j = 0; i < input_length;) { 42 | uint32_t sextet_a = data[i] == '=' ? 0 & i++ : k_decoding_table[data[i++]]; 43 | uint32_t sextet_b = data[i] == '=' ? 0 & i++ : k_decoding_table[data[i++]]; 44 | uint32_t sextet_c = data[i] == '=' ? 0 & i++ : k_decoding_table[data[i++]]; 45 | uint32_t sextet_d = data[i] == '=' ? 0 & i++ : k_decoding_table[data[i++]]; 46 | 47 | uint32_t triple = (sextet_a << 3 * 6) + (sextet_b << 2 * 6) + 48 | (sextet_c << 1 * 6) + (sextet_d << 0 * 6); 49 | 50 | if (j < output_length) { 51 | decoded_data[j++] = (triple >> 2 * 8) & 0xFF; 52 | } 53 | if (j < output_length) { 54 | decoded_data[j++] = (triple >> 1 * 8) & 0xFF; 55 | } 56 | if (j < output_length) { 57 | decoded_data[j++] = (triple >> 0 * 8) & 0xFF; 58 | } 59 | } 60 | 61 | return decoded_data; 62 | } 63 | 64 | } // namespace duckdb 65 | -------------------------------------------------------------------------------- /src/utils/serialization.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/serialization.hpp" 2 | 3 | #include "duckdb/common/serializer/deserializer.hpp" 4 | #include "duckdb/common/serializer/serializer.hpp" 5 | 6 | namespace duckdb { 7 | namespace ui { 8 | 9 | void EmptyResult::Serialize(Serializer &) const {} 10 | 11 | void TokenizeResult::Serialize(Serializer &serializer) const { 12 | serializer.WriteProperty(100, "offsets", offsets); 13 | serializer.WriteProperty(101, "types", types); 14 | } 15 | 16 | // Adapted from parts of DataChunk::Serialize 17 | void ColumnNamesAndTypes::Serialize(Serializer &serializer) const { 18 | serializer.WriteProperty(100, "names", names); 19 | serializer.WriteProperty(101, "types", types); 20 | } 21 | 22 | // Adapted from parts of DataChunk::Serialize 23 | void Chunk::Serialize(Serializer &serializer) const { 24 | serializer.WriteProperty(100, "row_count", row_count); 25 | serializer.WriteList(101, "vectors", vectors.size(), 26 | [&](Serializer::List &list, idx_t i) { 27 | list.WriteObject([&](Serializer &object) { 28 | // Reference the vector to avoid potentially mutating 29 | // it during serialization 30 | Vector serialized_vector(vectors[i].GetType()); 31 | serialized_vector.Reference(vectors[i]); 32 | serialized_vector.Serialize(object, row_count); 33 | }); 34 | }); 35 | } 36 | 37 | void SuccessResult::Serialize(Serializer &serializer) const { 38 | serializer.WriteProperty(100, "success", true); 39 | serializer.WriteProperty(101, "column_names_and_types", 40 | column_names_and_types); 41 | serializer.WriteList( 42 | 102, "chunks", chunks.size(), 43 | [&](Serializer::List &list, idx_t i) { list.WriteElement(chunks[i]); }); 44 | } 45 | 46 | void ErrorResult::Serialize(Serializer &serializer) const { 47 | serializer.WriteProperty(100, "success", false); 48 | serializer.WriteProperty(101, "error", error); 49 | } 50 | 51 | } // namespace ui 52 | } // namespace duckdb 53 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/conversion/functions/dataViewReaders.ts: -------------------------------------------------------------------------------- 1 | // DuckDB's physical storage and binary serialization format is little endian. 2 | const littleEndian = true; 3 | 4 | export function getInt8(dataView: DataView, offset: number): number { 5 | return dataView.getInt8(offset); 6 | } 7 | 8 | export function getUInt8(dataView: DataView, offset: number): number { 9 | return dataView.getUint8(offset); 10 | } 11 | 12 | export function getInt16(dataView: DataView, offset: number): number { 13 | return dataView.getInt16(offset, littleEndian); 14 | } 15 | 16 | export function getUInt16(dataView: DataView, offset: number): number { 17 | return dataView.getUint16(offset, littleEndian); 18 | } 19 | 20 | export function getInt32(dataView: DataView, offset: number): number { 21 | return dataView.getInt32(offset, littleEndian); 22 | } 23 | 24 | export function getUInt32(dataView: DataView, offset: number): number { 25 | return dataView.getUint32(offset, littleEndian); 26 | } 27 | 28 | export function getInt64(dataView: DataView, offset: number): bigint { 29 | return dataView.getBigInt64(offset, littleEndian); 30 | } 31 | 32 | export function getUInt64(dataView: DataView, offset: number): bigint { 33 | return dataView.getBigUint64(offset, littleEndian); 34 | } 35 | 36 | export function getFloat32(dataView: DataView, offset: number): number { 37 | return dataView.getFloat32(offset, littleEndian); 38 | } 39 | 40 | export function getFloat64(dataView: DataView, offset: number): number { 41 | return dataView.getFloat64(offset, littleEndian); 42 | } 43 | 44 | export function getInt128(dataView: DataView, offset: number): bigint { 45 | const lower = getUInt64(dataView, offset); 46 | const upper = getInt64(dataView, offset + 8); 47 | return (upper << BigInt(64)) + lower; 48 | } 49 | 50 | export function getUInt128(dataView: DataView, offset: number): bigint { 51 | const lower = getUInt64(dataView, offset); 52 | const upper = getUInt64(dataView, offset + 8); 53 | return (BigInt.asUintN(64, upper) << BigInt(64)) | BigInt.asUintN(64, lower); 54 | } 55 | 56 | export function getBoolean(dataView: DataView, offset: number): boolean { 57 | return getUInt8(dataView, offset) !== 0; 58 | } 59 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/conversion/displayStringForDuckDBValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { displayStringForDuckDBValue } from '../../src/conversion/displayStringForDuckDBValue'; 3 | 4 | // tests primitives 5 | suite('displayStringForDuckDBValue', () => { 6 | suite('null', () => { 7 | test('should convert null to NULL', () => { 8 | expect(displayStringForDuckDBValue(null)).toStrictEqual('NULL'); 9 | }); 10 | }); 11 | 12 | suite('string', () => { 13 | test('should wrap simple string in single quotes', () => { 14 | expect(displayStringForDuckDBValue('hello')).toStrictEqual("'hello'"); 15 | }); 16 | 17 | test('should escape single quotes by doubling them', () => { 18 | expect(displayStringForDuckDBValue("it's")).toStrictEqual("'it''s'"); 19 | }); 20 | 21 | test('should handle multiple single quotes', () => { 22 | expect(displayStringForDuckDBValue("don't say 'goose'")).toStrictEqual( 23 | "'don''t say ''goose'''", 24 | ); 25 | }); 26 | 27 | test('should wrap empty string in single quotes', () => { 28 | expect(displayStringForDuckDBValue('')).toStrictEqual("''"); 29 | }); 30 | }); 31 | 32 | suite('number', () => { 33 | test('should convert integer to string', () => { 34 | expect(displayStringForDuckDBValue(123)).toStrictEqual('123'); 35 | }); 36 | 37 | test('should convert float to string', () => { 38 | expect(displayStringForDuckDBValue(123.456)).toStrictEqual('123.456'); 39 | }); 40 | 41 | test('should convert negative number to string', () => { 42 | expect(displayStringForDuckDBValue(-42)).toStrictEqual('-42'); 43 | }); 44 | }); 45 | 46 | suite('boolean', () => { 47 | test('should convert true to string', () => { 48 | expect(displayStringForDuckDBValue(true)).toStrictEqual('true'); 49 | }); 50 | 51 | test('should convert false to string', () => { 52 | expect(displayStringForDuckDBValue(false)).toStrictEqual('false'); 53 | }); 54 | }); 55 | 56 | suite('bigint', () => { 57 | test('should convert bigint to string', () => { 58 | expect(displayStringForDuckDBValue(123n)).toStrictEqual('123'); 59 | }); 60 | 61 | test('should convert large bigint to string', () => { 62 | expect(displayStringForDuckDBValue(9007199254740991n)).toStrictEqual( 63 | '9007199254740991', 64 | ); 65 | }); 66 | }); 67 | }); 68 | -------------------------------------------------------------------------------- /src/event_dispatcher.cpp: -------------------------------------------------------------------------------- 1 | #include "event_dispatcher.hpp" 2 | 3 | #include 4 | 5 | #define CPPHTTPLIB_OPENSSL_SUPPORT 6 | #include "httplib.hpp" 7 | 8 | namespace httplib = duckdb_httplib_openssl; 9 | 10 | // Chosen to be no more than half of the lesser of the two limits: 11 | // - The default httplib thread pool size = 8 12 | // - The browser limit on the number of server-sent event connections = 6 13 | #define MAX_EVENT_WAIT_COUNT 3 14 | 15 | namespace duckdb { 16 | namespace ui { 17 | // An empty Server-Sent Events message. See 18 | // https://html.spec.whatwg.org/multipage/server-sent-events.html#authoring-notes 19 | constexpr const char *EMPTY_SSE_MESSAGE = ":\r\r"; 20 | constexpr idx_t EMPTY_SSE_MESSAGE_LENGTH = 3; 21 | 22 | bool EventDispatcher::WaitEvent(httplib::DataSink *sink) { 23 | std::unique_lock lock(mutex); 24 | // Don't allow too many simultaneous waits, because each consumes a thread in 25 | // the httplib thread pool, and also browsers limit the number of server-sent 26 | // event connections. 27 | if (closed || wait_count >= MAX_EVENT_WAIT_COUNT) { 28 | return false; 29 | } 30 | int target_id = next_id; 31 | wait_count++; 32 | cv.wait_for(lock, std::chrono::seconds(5)); 33 | wait_count--; 34 | if (closed) { 35 | return false; 36 | } 37 | if (current_id == target_id) { 38 | sink->write(message.data(), message.size()); 39 | } else { 40 | // Our wait timer expired. Write an empty, no-op message. 41 | // This enables detecting when the client is gone. 42 | sink->write(EMPTY_SSE_MESSAGE, EMPTY_SSE_MESSAGE_LENGTH); 43 | } 44 | return true; 45 | } 46 | 47 | void EventDispatcher::SendEvent(const std::string &_message) { 48 | std::lock_guard guard(mutex); 49 | if (closed) { 50 | return; 51 | } 52 | 53 | current_id = next_id++; 54 | message = _message; 55 | cv.notify_all(); 56 | } 57 | 58 | void EventDispatcher::SendConnectedEvent(const std::string &token) { 59 | SendEvent(StringUtil::Format("event: ConnectedEvent\ndata: %s\n\n", token)); 60 | } 61 | 62 | void EventDispatcher::SendCatalogChangedEvent() { 63 | SendEvent("event: CatalogChangeEvent\ndata:\n\n"); 64 | } 65 | 66 | void EventDispatcher::Close() { 67 | std::lock_guard guard(mutex); 68 | if (closed) { 69 | return; 70 | } 71 | 72 | current_id = next_id++; 73 | closed = true; 74 | cv.notify_all(); 75 | } 76 | } // namespace ui 77 | } // namespace duckdb 78 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBTimestampNanosecondsValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBTimestampNanosecondsValue } from '../src/DuckDBTimestampNanosecondsValue'; 3 | 4 | suite('DuckDBTimestampNanosecondsValue', () => { 5 | test('should render a normal timestamp value to the correct string', () => { 6 | expect( 7 | new DuckDBTimestampNanosecondsValue(1612325106007891000n).toString(), 8 | ).toStrictEqual('2021-02-03 04:05:06.007891'); 9 | }); 10 | test('should render a zero timestamp value to the correct string', () => { 11 | expect(new DuckDBTimestampNanosecondsValue(0n).toString()).toStrictEqual( 12 | '1970-01-01 00:00:00', 13 | ); 14 | }); 15 | test('should render a negative timestamp value to the correct string', () => { 16 | expect( 17 | new DuckDBTimestampNanosecondsValue(-7000n).toString(), 18 | ).toStrictEqual('1969-12-31 23:59:59.999993'); 19 | }); 20 | test('should render a large positive timestamp value to the correct string', () => { 21 | expect( 22 | new DuckDBTimestampNanosecondsValue(8857641599999123000n).toString(), 23 | ).toStrictEqual('2250-09-08 23:59:59.999123'); 24 | }); 25 | test('should render a large negative timestamp value to the correct string', () => { 26 | expect( 27 | new DuckDBTimestampNanosecondsValue(-8495881076543211000n).toString(), 28 | ).toStrictEqual('1700-10-11 01:02:03.456789'); 29 | }); 30 | test('should render the max timestamp value to the correct string', () => { 31 | expect( 32 | new DuckDBTimestampNanosecondsValue(9223372036854775806n).toString(), 33 | ).toStrictEqual('2262-04-11 23:47:16.854775'); 34 | }); 35 | test('should render the min timestamp value to the correct string', () => { 36 | expect( 37 | new DuckDBTimestampNanosecondsValue(-9223372036854775806n).toString(), 38 | ).toStrictEqual('1677-09-21 00:12:43.145225'); 39 | }); 40 | 41 | suite('toSql', () => { 42 | test('should render timestamp to SQL', () => { 43 | const timestamp = new DuckDBTimestampNanosecondsValue( 44 | BigInt(1697212800) * 1000000000n, 45 | ); 46 | expect(timestamp.toSql()).toMatch(/^TIMESTAMP_NS '.+'$/); 47 | }); 48 | 49 | test('should render epoch to SQL', () => { 50 | expect(new DuckDBTimestampNanosecondsValue(0n).toSql()).toStrictEqual( 51 | "TIMESTAMP_NS '1970-01-01 00:00:00'", 52 | ); 53 | }); 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBBitValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBBitValue } from '../src/DuckDBBitValue'; 3 | 4 | suite('DuckDBBitValue', () => { 5 | test('should render an empty byte array to the correct string', () => { 6 | expect(new DuckDBBitValue(new Uint8Array([])).toString()).toStrictEqual(''); 7 | }); 8 | test('should render bit string with no padding to the correct string', () => { 9 | expect( 10 | new DuckDBBitValue(new Uint8Array([0x00, 0xf1, 0xe2, 0xd3])).toString(), 11 | ).toStrictEqual('111100011110001011010011'); 12 | }); 13 | test('should render bit string with padding to the correct string', () => { 14 | expect( 15 | new DuckDBBitValue(new Uint8Array([0x03, 0xf1, 0xe2, 0xd3])).toString(), 16 | ).toStrictEqual('100011110001011010011'); 17 | }); 18 | test('should round-trip bit string with no padding', () => { 19 | expect( 20 | DuckDBBitValue.fromString('111100011110001011010011').toString(), 21 | ).toStrictEqual('111100011110001011010011'); 22 | }); 23 | test('should round-trip bit string with padding', () => { 24 | expect( 25 | DuckDBBitValue.fromString('100011110001011010011').toString(), 26 | ).toStrictEqual('100011110001011010011'); 27 | }); 28 | test('toJson', () => { 29 | expect( 30 | DuckDBBitValue.fromString('100011110001011010011').toJson(), 31 | ).toStrictEqual('100011110001011010011'); 32 | }); 33 | 34 | suite('toSql', () => { 35 | test('should render bit string to SQL', () => { 36 | const bitValue = DuckDBBitValue.fromString('10101'); 37 | expect(bitValue.toSql()).toStrictEqual("'10101'::BITSTRING"); 38 | }); 39 | 40 | test('should render empty bit string to SQL', () => { 41 | const bitValue = DuckDBBitValue.fromString(''); 42 | expect(bitValue.toSql()).toStrictEqual("''::BITSTRING"); 43 | }); 44 | 45 | test('should render bit string with no padding to SQL', () => { 46 | const bitValue = DuckDBBitValue.fromString('111100011110001011010011'); 47 | expect(bitValue.toSql()).toStrictEqual( 48 | "'111100011110001011010011'::BITSTRING", 49 | ); 50 | }); 51 | 52 | test('should render bit string with padding to SQL', () => { 53 | const bitValue = DuckDBBitValue.fromString('100011110001011010011'); 54 | expect(bitValue.toSql()).toStrictEqual( 55 | "'100011110001011010011'::BITSTRING", 56 | ); 57 | }); 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/http/functions/makeDuckDBUIHttpRequestHeaders.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBUIRunOptions } from '../../client/types/DuckDBUIRunOptions.js'; 2 | import { toBase64 } from '../../util/functions/toBase64.js'; 3 | 4 | export interface DuckDBUIHttpRequestHeaderOptions extends DuckDBUIRunOptions { 5 | connectionName?: string; 6 | } 7 | 8 | export function makeDuckDBUIHttpRequestHeaders({ 9 | description, 10 | connectionName, 11 | databaseName, 12 | schemaName, 13 | errorsAsJson, 14 | parameters, 15 | resultRowLimit, 16 | resultDatabaseName, 17 | resultSchemaName, 18 | resultTableName, 19 | resultTableRowLimit, 20 | }: DuckDBUIHttpRequestHeaderOptions): Headers { 21 | const headers = new Headers(); 22 | // We base64 encode some values because they can contain characters invalid in an HTTP header. 23 | if (description) { 24 | headers.append('X-DuckDB-UI-Request-Description', description); 25 | } 26 | if (connectionName) { 27 | headers.append('X-DuckDB-UI-Connection-Name', connectionName); 28 | } 29 | if (databaseName) { 30 | headers.append('X-DuckDB-UI-Database-Name', toBase64(databaseName)); 31 | } 32 | if (schemaName) { 33 | headers.append('X-DuckDB-UI-Schema-Name', toBase64(schemaName)); 34 | } 35 | if (parameters) { 36 | headers.append('X-DuckDB-UI-Parameter-Count', String(parameters.length)); 37 | for (let i = 0; i < parameters.length; i++) { 38 | // TODO: support non-string parameters? 39 | headers.append( 40 | `X-DuckDB-UI-Parameter-Value-${i}`, 41 | toBase64(String(parameters[i])), 42 | ); 43 | } 44 | } 45 | if (resultRowLimit !== undefined) { 46 | headers.append('X-DuckDB-UI-Result-Row-Limit', String(resultRowLimit)); 47 | } 48 | if (resultDatabaseName) { 49 | headers.append( 50 | 'X-DuckDB-UI-Result-Database-Name', 51 | toBase64(resultDatabaseName), 52 | ); 53 | } 54 | if (resultSchemaName) { 55 | headers.append( 56 | 'X-DuckDB-UI-Result-Schema-Name', 57 | toBase64(resultSchemaName), 58 | ); 59 | } 60 | if (resultTableName) { 61 | headers.append('X-DuckDB-UI-Result-Table-Name', toBase64(resultTableName)); 62 | } 63 | if (resultTableRowLimit !== undefined) { 64 | headers.append( 65 | 'X-DuckDB-UI-Result-Table-Row-Limit', 66 | String(resultTableRowLimit), 67 | ); 68 | } 69 | if (errorsAsJson) { 70 | headers.append('X-DuckDB-UI-Errors-As-JSON', 'true'); 71 | } 72 | return headers; 73 | } 74 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBUUIDValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBUUIDValue } from '../src/DuckDBUUIDValue'; 3 | 4 | suite('DuckDBUUIDValue', () => { 5 | test('should render all zero bytes to the correct string', () => { 6 | expect( 7 | new DuckDBUUIDValue( 8 | new Uint8Array([ 9 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 10 | 0x00, 0x00, 0x00, 0x00, 0x00, 11 | ]), 12 | ).toString(), 13 | ).toStrictEqual('00000000-0000-0000-0000-000000000000'); 14 | }); 15 | test('should render all max bytes to the correct string', () => { 16 | expect( 17 | new DuckDBUUIDValue( 18 | new Uint8Array([ 19 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 20 | 0xff, 0xff, 0xff, 0xff, 0xff, 21 | ]), 22 | ).toString(), 23 | ).toStrictEqual('ffffffff-ffff-ffff-ffff-ffffffffffff'); 24 | }); 25 | test('should render arbitrary bytes to the correct string', () => { 26 | expect( 27 | new DuckDBUUIDValue( 28 | new Uint8Array([ 29 | 0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87, 0xfe, 0xdc, 0xba, 30 | 0x98, 0x76, 0x54, 0x32, 0x10, 31 | ]), 32 | ).toString(), 33 | ).toStrictEqual('f0e1d2c3-b4a5-9687-fedc-ba9876543210'); 34 | }); 35 | test('should render a uint128 to the correct string', () => { 36 | expect( 37 | DuckDBUUIDValue.fromUint128( 38 | 0xf0e1d2c3b4a59687fedcba9876543210n, 39 | ).toString(), 40 | ).toStrictEqual('f0e1d2c3-b4a5-9687-fedc-ba9876543210'); 41 | }); 42 | test('should render a stored hugeint to the correct string', () => { 43 | expect( 44 | DuckDBUUIDValue.fromStoredHugeint( 45 | 0x70e1d2c3b4a59687fedcba9876543210n, // note the flipped MSB 46 | ).toString(), 47 | ).toStrictEqual('f0e1d2c3-b4a5-9687-fedc-ba9876543210'); 48 | }); 49 | 50 | suite('toSql', () => { 51 | test('should render UUID to SQL', () => { 52 | const uuid = 53 | DuckDBUUIDValue.fromUint128(0x123e4567e89b12d3a456426614174000n); 54 | expect(uuid.toSql()).toMatch(/^'.{36}'::UUID$/); 55 | }); 56 | 57 | test('should render UUID with proper format to SQL', () => { 58 | const uuid = 59 | DuckDBUUIDValue.fromUint128(0x550e8400e29b41d4a716446655440000n); 60 | expect(uuid.toSql()).toStrictEqual( 61 | "'550e8400-e29b-41d4-a716-446655440000'::UUID", 62 | ); 63 | }); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBTimestampSecondsValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBTimestampSecondsValue } from '../src/DuckDBTimestampSecondsValue'; 3 | 4 | suite('DuckDBTimestampSecondsValue', () => { 5 | test('should render a normal timestamp value to the correct string', () => { 6 | expect( 7 | new DuckDBTimestampSecondsValue(1612325106n).toString(), 8 | ).toStrictEqual('2021-02-03 04:05:06'); 9 | }); 10 | test('should render a zero timestamp value to the correct string', () => { 11 | expect(new DuckDBTimestampSecondsValue(0n).toString()).toStrictEqual( 12 | '1970-01-01 00:00:00', 13 | ); 14 | }); 15 | test('should render a negative timestamp value to the correct string', () => { 16 | expect(new DuckDBTimestampSecondsValue(-7n).toString()).toStrictEqual( 17 | '1969-12-31 23:59:53', 18 | ); 19 | }); 20 | test('should render a large positive timestamp value to the correct string', () => { 21 | expect( 22 | new DuckDBTimestampSecondsValue(2353318271999n).toString(), 23 | ).toStrictEqual('76543-09-08 23:59:59'); 24 | }); 25 | test('should render a large negative (AD) timestamp value to the correct string', () => { 26 | expect( 27 | new DuckDBTimestampSecondsValue(-58261244277n).toString(), 28 | ).toStrictEqual('0123-10-11 01:02:03'); 29 | }); 30 | test('should render a large negative (BC) timestamp value to the correct string', () => { 31 | expect( 32 | new DuckDBTimestampSecondsValue(-65992661877n).toString(), 33 | ).toStrictEqual('0123-10-11 (BC) 01:02:03'); 34 | }); 35 | test('should render the max timestamp value to the correct string', () => { 36 | expect( 37 | new DuckDBTimestampSecondsValue(9223372036854n).toString(), 38 | ).toStrictEqual('294247-01-10 04:00:54'); 39 | }); 40 | test('should render the min timestamp value to the correct string', () => { 41 | expect( 42 | new DuckDBTimestampSecondsValue(-9223372022400n).toString(), 43 | ).toStrictEqual('290309-12-22 (BC) 00:00:00'); 44 | }); 45 | 46 | suite('toSql', () => { 47 | test('should render timestamp to SQL', () => { 48 | const timestamp = new DuckDBTimestampSecondsValue(BigInt(1697212800)); 49 | expect(timestamp.toSql()).toMatch(/^TIMESTAMP_S '.+'$/); 50 | }); 51 | 52 | test('should render epoch to SQL', () => { 53 | expect(new DuckDBTimestampSecondsValue(0n).toSql()).toStrictEqual( 54 | "TIMESTAMP_S '1970-01-01 00:00:00'", 55 | ); 56 | }); 57 | }); 58 | }); 59 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/client/classes/DuckDBUIClient.ts: -------------------------------------------------------------------------------- 1 | import { sendDuckDBUIHttpRequest } from '../../http/functions/sendDuckDBUIHttpRequest.js'; 2 | import { tokenizeResultFromBuffer } from '../../serialization/functions/tokenizeResultFromBuffer.js'; 3 | import type { TokenizeResult } from '../../serialization/types/TokenizeResult.js'; 4 | import { DuckDBUIClientConnection } from './DuckDBUIClientConnection.js'; 5 | 6 | export { DuckDBUIClientConnection }; 7 | export type { TokenizeResult }; 8 | 9 | export class DuckDBUIClient { 10 | private readonly eventSource: EventSource; 11 | 12 | private defaultConnection: DuckDBUIClientConnection | undefined; 13 | 14 | private constructor() { 15 | this.eventSource = new EventSource('/localEvents'); 16 | } 17 | 18 | public addOpenEventListener(listener: (event: Event) => void) { 19 | this.eventSource.addEventListener('open', listener); 20 | } 21 | 22 | public removeOpenEventListener(listener: (event: Event) => void) { 23 | this.eventSource.removeEventListener('open', listener); 24 | } 25 | 26 | public addErrorEventListener(listener: (event: Event) => void) { 27 | this.eventSource.addEventListener('error', listener); 28 | } 29 | 30 | public removeErrorEventListener(listener: (event: Event) => void) { 31 | this.eventSource.removeEventListener('error', listener); 32 | } 33 | 34 | public addMessageEventListener( 35 | type: string, 36 | listener: (event: MessageEvent) => void, 37 | ) { 38 | this.eventSource.addEventListener(type, listener); 39 | } 40 | 41 | public removeMessageEventListener( 42 | type: string, 43 | listener: (event: MessageEvent) => void, 44 | ) { 45 | this.eventSource.removeEventListener(type, listener); 46 | } 47 | 48 | public connect() { 49 | return new DuckDBUIClientConnection(); 50 | } 51 | 52 | public get connection(): DuckDBUIClientConnection { 53 | if (!this.defaultConnection) { 54 | this.defaultConnection = this.connect(); 55 | } 56 | return this.defaultConnection; 57 | } 58 | 59 | public async tokenize(text: string): Promise { 60 | const buffer = await sendDuckDBUIHttpRequest('/ddb/tokenize', text); 61 | return tokenizeResultFromBuffer(buffer); 62 | } 63 | 64 | private static singletonInstance: DuckDBUIClient; 65 | 66 | public static get singleton(): DuckDBUIClient { 67 | if (!DuckDBUIClient.singletonInstance) { 68 | DuckDBUIClient.singletonInstance = new DuckDBUIClient(); 69 | } 70 | return DuckDBUIClient.singletonInstance; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5...3.31.5) 2 | 3 | # Set extension name here 4 | set(TARGET_NAME ui) 5 | 6 | # DuckDB's extension distribution supports vcpkg. As such, dependencies can be 7 | # added in ./vcpkg.json and then used in cmake with find_package. Feel free to 8 | # remove or replace with other dependencies. Note that it should also be removed 9 | # from vcpkg.json to prevent needlessly installing it.. 10 | find_package(OpenSSL REQUIRED) 11 | 12 | include_directories(${OPENSSL_INCLUDE_DIR}) 13 | 14 | set(EXTENSION_NAME ${TARGET_NAME}_extension) 15 | 16 | project(${TARGET_NAME}) 17 | include_directories(src/include ${PROJECT_SOURCE_DIR}/third_party/httplib) 18 | 19 | set(EXTENSION_SOURCES 20 | src/event_dispatcher.cpp 21 | src/http_server.cpp 22 | src/settings.cpp 23 | src/state.cpp 24 | src/ui_extension.cpp 25 | src/utils/encoding.cpp 26 | src/utils/env.cpp 27 | src/utils/helpers.cpp 28 | src/utils/md_helpers.cpp 29 | src/utils/serialization.cpp 30 | src/watcher.cpp) 31 | 32 | add_definitions(-DDUCKDB_MAJOR_VERSION=${DUCKDB_MAJOR_VERSION}) 33 | add_definitions(-DDUCKDB_MINOR_VERSION=${DUCKDB_MINOR_VERSION}) 34 | add_definitions(-DDUCKDB_PATCH_VERSION=${DUCKDB_PATCH_VERSION}) 35 | 36 | find_package(Git) 37 | if(NOT Git_FOUND) 38 | message(FATAL_ERROR "Git not found, unable to determine git sha") 39 | endif() 40 | 41 | execute_process( 42 | COMMAND ${GIT_EXECUTABLE} rev-list --count HEAD 43 | WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} 44 | OUTPUT_VARIABLE UI_EXTENSION_SEQ_NUM 45 | OUTPUT_STRIP_TRAILING_WHITESPACE) 46 | 47 | message(STATUS "UI_EXTENSION_SEQ_NUM=${UI_EXTENSION_SEQ_NUM}") 48 | add_definitions(-DUI_EXTENSION_SEQ_NUM="${UI_EXTENSION_SEQ_NUM}") 49 | 50 | execute_process( 51 | COMMAND ${GIT_EXECUTABLE} rev-parse --short=10 HEAD 52 | WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} 53 | OUTPUT_VARIABLE UI_EXTENSION_GIT_SHA 54 | OUTPUT_STRIP_TRAILING_WHITESPACE) 55 | 56 | message(STATUS "UI_EXTENSION_GIT_SHA=${UI_EXTENSION_GIT_SHA}") 57 | add_definitions(-DUI_EXTENSION_GIT_SHA="${UI_EXTENSION_GIT_SHA}") 58 | 59 | build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) 60 | build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) 61 | 62 | target_link_libraries(${EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto) 63 | target_link_libraries(${TARGET_NAME}_loadable_extension OpenSSL::SSL OpenSSL::Crypto) 64 | 65 | install( 66 | TARGETS ${EXTENSION_NAME} 67 | EXPORT "${DUCKDB_EXPORT_SET}" 68 | LIBRARY DESTINATION "${INSTALL_LIB_DIR}" 69 | ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") 70 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBTimestampMillisecondsValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBTimestampMillisecondsValue } from '../src/DuckDBTimestampMillisecondsValue'; 3 | 4 | suite('DuckDBTimestampMillisecondsValue', () => { 5 | test('should render a normal timestamp value to the correct string', () => { 6 | expect( 7 | new DuckDBTimestampMillisecondsValue(1612325106007n).toString(), 8 | ).toStrictEqual('2021-02-03 04:05:06.007'); 9 | }); 10 | test('should render a zero timestamp value to the correct string', () => { 11 | expect(new DuckDBTimestampMillisecondsValue(0n).toString()).toStrictEqual( 12 | '1970-01-01 00:00:00', 13 | ); 14 | }); 15 | test('should render a negative timestamp value to the correct string', () => { 16 | expect(new DuckDBTimestampMillisecondsValue(-7n).toString()).toStrictEqual( 17 | '1969-12-31 23:59:59.993', 18 | ); 19 | }); 20 | test('should render a large positive timestamp value to the correct string', () => { 21 | expect( 22 | new DuckDBTimestampMillisecondsValue(2353318271999999n).toString(), 23 | ).toStrictEqual('76543-09-08 23:59:59.999'); 24 | }); 25 | test('should render a large negative (AD) timestamp value to the correct string', () => { 26 | expect( 27 | new DuckDBTimestampMillisecondsValue(-58261244276544n).toString(), 28 | ).toStrictEqual('0123-10-11 01:02:03.456'); 29 | }); 30 | test('should render a large negative (BC) timestamp value to the correct string', () => { 31 | expect( 32 | new DuckDBTimestampMillisecondsValue(-65992661876544n).toString(), 33 | ).toStrictEqual('0123-10-11 (BC) 01:02:03.456'); 34 | }); 35 | test('should render the max timestamp value to the correct string', () => { 36 | expect( 37 | new DuckDBTimestampMillisecondsValue(9223372036854775n).toString(), 38 | ).toStrictEqual('294247-01-10 04:00:54.775'); 39 | }); 40 | test('should render the min timestamp value to the correct string', () => { 41 | expect( 42 | new DuckDBTimestampMillisecondsValue(-9223372022400000n).toString(), 43 | ).toStrictEqual('290309-12-22 (BC) 00:00:00'); 44 | }); 45 | 46 | suite('toSql', () => { 47 | test('should render timestamp to SQL', () => { 48 | const timestamp = new DuckDBTimestampMillisecondsValue( 49 | BigInt(1697212800) * 1000n, 50 | ); 51 | expect(timestamp.toSql()).toMatch(/^TIMESTAMP_MS '.+'$/); 52 | }); 53 | 54 | test('should render epoch to SQL', () => { 55 | expect(new DuckDBTimestampMillisecondsValue(0n).toSql()).toStrictEqual( 56 | "TIMESTAMP_MS '1970-01-01 00:00:00'", 57 | ); 58 | }); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/client/classes/DuckDBUIClientConnection.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBUIHttpRequestQueue } from '../../http/classes/DuckDBUIHttpRequestQueue.js'; 2 | import { makeDuckDBUIHttpRequestHeaders } from '../../http/functions/makeDuckDBUIHttpRequestHeaders.js'; 3 | import { sendDuckDBUIHttpRequest } from '../../http/functions/sendDuckDBUIHttpRequest.js'; 4 | import { randomString } from '../../util/functions/randomString.js'; 5 | import { materializedRunResultFromQueueResult } from '../functions/materializedRunResultFromQueueResult.js'; 6 | import { DuckDBUIRunOptions } from '../types/DuckDBUIRunOptions.js'; 7 | import { MaterializedRunResult } from '../types/MaterializedRunResult.js'; 8 | 9 | export class DuckDBUIClientConnection { 10 | private readonly connectionName = `connection_${randomString()}`; 11 | 12 | private readonly requestQueue: DuckDBUIHttpRequestQueue = 13 | new DuckDBUIHttpRequestQueue(); 14 | 15 | public async run( 16 | sql: string, 17 | options?: DuckDBUIRunOptions, 18 | ): Promise { 19 | const queueResult = await this.requestQueue.enqueueAndWait( 20 | '/ddb/run', 21 | sql, 22 | this.makeHeaders(options), 23 | ); 24 | return materializedRunResultFromQueueResult(queueResult); 25 | } 26 | 27 | public enqueue(sql: string, options?: DuckDBUIRunOptions): string { 28 | return this.requestQueue.enqueue( 29 | '/ddb/run', 30 | sql, 31 | this.makeHeaders(options), 32 | ); 33 | } 34 | 35 | public cancel( 36 | id: string, 37 | errorMessage?: string, 38 | failure?: (reason: unknown) => void, 39 | ) { 40 | // Handle the rejected promise (with a no-op) in case nothing else is, to avoid a console error. 41 | this.requestQueue.enqueuedResult(id).catch(() => {}); 42 | this.requestQueue.cancel(id, errorMessage); 43 | // If currently running, then interrupt it. 44 | if (this.requestQueue.isCurrent(id)) { 45 | // Don't await (but report any unexpected errors). Canceling should return synchronously. 46 | sendDuckDBUIHttpRequest('/ddb/interrupt', '', this.makeHeaders()).catch( 47 | failure, 48 | ); 49 | } 50 | return true; 51 | } 52 | 53 | public async enqueuedResult(id: string): Promise { 54 | const queueResult = await this.requestQueue.enqueuedResult(id); 55 | return materializedRunResultFromQueueResult(queueResult); 56 | } 57 | 58 | public get enqueuedCount(): number { 59 | return this.requestQueue.length; 60 | } 61 | 62 | private makeHeaders(options: DuckDBUIRunOptions = {}): Headers { 63 | return makeDuckDBUIHttpRequestHeaders({ 64 | ...options, 65 | connectionName: this.connectionName, 66 | }); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBArrayValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBArrayValue } from '../src/DuckDBArrayValue'; 3 | import { DuckDBMapValue } from '../src/DuckDBMapValue'; 4 | 5 | suite('DuckDBArrayValue', () => { 6 | test('should render an empty array to the correct string', () => { 7 | expect(new DuckDBArrayValue([]).toString()).toStrictEqual('[]'); 8 | }); 9 | test('should render a single element array to the correct string', () => { 10 | expect(new DuckDBArrayValue([123]).toString()).toStrictEqual('[123]'); 11 | }); 12 | test('should render a multi-element array to the correct string', () => { 13 | expect( 14 | new DuckDBArrayValue(['abc', null, true, '']).toString(), 15 | ).toStrictEqual(`['abc', NULL, true, '']`); 16 | }); 17 | test('should render an array with nested arrays to the correct string', () => { 18 | expect( 19 | new DuckDBArrayValue([ 20 | new DuckDBArrayValue([]), 21 | null, 22 | new DuckDBArrayValue([123, null, 'xyz']), 23 | ]).toString(), 24 | ).toStrictEqual(`[[], NULL, [123, NULL, 'xyz']]`); 25 | }); 26 | test('toJson array with basic values', () => { 27 | expect(new DuckDBArrayValue([123, 'abc', null]).toJson()).toStrictEqual([ 28 | 123, 29 | 'abc', 30 | null, 31 | ]); 32 | }); 33 | test('toJson array with complex values', () => { 34 | expect( 35 | new DuckDBArrayValue([ 36 | new DuckDBMapValue([ 37 | { key: 'foo', value: 123 }, 38 | { key: 'bar', value: 'abc' }, 39 | ]), 40 | new DuckDBArrayValue([123, null, 'xyz']), 41 | null, 42 | ]).toJson(), 43 | ).toStrictEqual([ 44 | { "'foo'": 123, "'bar'": 'abc' }, 45 | [123, null, 'xyz'], 46 | null, 47 | ]); 48 | }); 49 | 50 | suite('toSql', () => { 51 | test('should render empty array', () => { 52 | expect(new DuckDBArrayValue([]).toSql()).toStrictEqual('[]'); 53 | }); 54 | 55 | test('should render array with numbers', () => { 56 | expect(new DuckDBArrayValue([1, 2, 3]).toSql()).toStrictEqual( 57 | '[1, 2, 3]', 58 | ); 59 | }); 60 | 61 | test('should render array with strings', () => { 62 | expect(new DuckDBArrayValue(['a', 'b', 'c']).toSql()).toStrictEqual( 63 | "['a', 'b', 'c']", 64 | ); 65 | }); 66 | 67 | test('should render array with null', () => { 68 | expect(new DuckDBArrayValue([1, null, 3]).toSql()).toStrictEqual( 69 | '[1, NULL, 3]', 70 | ); 71 | }); 72 | 73 | test('should render nested arrays', () => { 74 | expect( 75 | new DuckDBArrayValue([ 76 | new DuckDBArrayValue([1, 2]), 77 | new DuckDBArrayValue([3, 4]), 78 | ]).toSql(), 79 | ).toStrictEqual('[[1, 2], [3, 4]]'); 80 | }); 81 | }); 82 | }); 83 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/conversion/functions/typeInfoGetters.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ArrayTypeInfo, 3 | DecimalTypeInfo, 4 | EnumTypeInfo, 5 | ListTypeInfo, 6 | StructTypeInfo, 7 | TypeIdAndInfo, 8 | TypeInfo, 9 | } from '../../serialization/types/TypeInfo.js'; 10 | 11 | export function getArrayTypeInfo( 12 | typeInfo: TypeInfo | undefined, 13 | ): ArrayTypeInfo { 14 | if (!typeInfo) { 15 | throw new Error(`ARRAY has no typeInfo!`); 16 | } 17 | if (typeInfo.kind !== 'array') { 18 | throw new Error(`ARRAY has unexpected typeInfo.kind: ${typeInfo.kind}`); 19 | } 20 | return typeInfo; 21 | } 22 | 23 | export function getDecimalTypeInfo( 24 | typeInfo: TypeInfo | undefined, 25 | ): DecimalTypeInfo { 26 | if (!typeInfo) { 27 | throw new Error(`DECIMAL has no typeInfo!`); 28 | } 29 | if (typeInfo.kind !== 'decimal') { 30 | throw new Error(`DECIMAL has unexpected typeInfo.kind: ${typeInfo.kind}`); 31 | } 32 | return typeInfo; 33 | } 34 | 35 | export function getEnumTypeInfo(typeInfo: TypeInfo | undefined): EnumTypeInfo { 36 | if (!typeInfo) { 37 | throw new Error(`ENUM has no typeInfo!`); 38 | } 39 | if (typeInfo.kind !== 'enum') { 40 | throw new Error(`ENUM has unexpected typeInfo.kind: ${typeInfo.kind}`); 41 | } 42 | return typeInfo; 43 | } 44 | 45 | export function getListTypeInfo(typeInfo: TypeInfo | undefined): ListTypeInfo { 46 | if (!typeInfo) { 47 | throw new Error(`LIST has no typeInfo!`); 48 | } 49 | if (typeInfo.kind !== 'list') { 50 | throw new Error(`LIST has unexpected typeInfo.kind: ${typeInfo.kind}`); 51 | } 52 | return typeInfo; 53 | } 54 | 55 | export function getStructTypeInfo( 56 | typeInfo: TypeInfo | undefined, 57 | ): StructTypeInfo { 58 | if (!typeInfo) { 59 | throw new Error(`STRUCT has no typeInfo!`); 60 | } 61 | if (typeInfo.kind !== 'struct') { 62 | throw new Error(`STRUCT has unexpected typeInfo.kind: ${typeInfo.kind}`); 63 | } 64 | return typeInfo; 65 | } 66 | 67 | export function getMapTypeInfos(typeInfo: TypeInfo | undefined): { 68 | keyType: TypeIdAndInfo; 69 | valueType: TypeIdAndInfo; 70 | } { 71 | // MAP = LIST(STRUCT(key KEY_TYPE, value VALUE_TYPE)) 72 | const { childType } = getListTypeInfo(typeInfo); 73 | const { childTypes } = getStructTypeInfo(childType.typeInfo); 74 | if (childTypes.length !== 2) { 75 | throw new Error( 76 | `MAP childType has unexpected childTypes length: ${childTypes.length}`, 77 | ); 78 | } 79 | if (childTypes[0].length !== 2) { 80 | throw new Error( 81 | `MAP childType has unexpected childTypes[0] length: ${childTypes[0].length}`, 82 | ); 83 | } 84 | if (childTypes[1].length !== 2) { 85 | throw new Error( 86 | `MAP childType has unexpected childTypes[1] length: ${childTypes[1].length}`, 87 | ); 88 | } 89 | return { 90 | keyType: childTypes[0][1], 91 | valueType: childTypes[1][1], 92 | }; 93 | } 94 | -------------------------------------------------------------------------------- /.github/workflows/MainDistributionPipeline.yml: -------------------------------------------------------------------------------- 1 | # 2 | # This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension 3 | # 4 | name: Main Extension Distribution Pipeline 5 | on: 6 | pull_request: 7 | paths-ignore: 8 | - ".github/workflows/TypeScriptWorkspace.yml" 9 | - "docs/**" 10 | - "ts/**" 11 | - "README.md" 12 | push: 13 | branches: 14 | - "main" 15 | paths-ignore: 16 | - ".github/workflows/TypeScriptWorkspace.yml" 17 | - "docs/**" 18 | - "ts/**" 19 | - "README.md" 20 | workflow_dispatch: 21 | 22 | concurrency: 23 | group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} 24 | cancel-in-progress: true 25 | 26 | jobs: 27 | duckdb-main-build: 28 | name: Build main extension binaries 29 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main 30 | with: 31 | ci_tools_version: main 32 | duckdb_version: main 33 | exclude_archs: ${{ github.repository == 'duckdb/duckdb-ui' && 'wasm_mvp;wasm_eh;wasm_threads' || 'linux_arm64;linux_amd64_musl;osx_amd64;windows_amd64_mingw;wasm_mvp;wasm_eh;wasm_threads' }} 34 | extension_name: ui 35 | 36 | duckdb-next-patch-build: 37 | name: Build next patch extension binaries 38 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.3 39 | with: 40 | ci_tools_version: v1.4.3 41 | duckdb_version: v1.4-andium 42 | exclude_archs: ${{ github.repository == 'duckdb/duckdb-ui' && 'wasm_mvp;wasm_eh;wasm_threads' || 'linux_arm64;linux_amd64_musl;osx_amd64;windows_amd64_mingw;wasm_mvp;wasm_eh;wasm_threads' }} 43 | extension_name: ui 44 | 45 | duckdb-stable-build: 46 | name: Build UI extension for DuckDB v1.4.3 47 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.3 48 | with: 49 | ci_tools_version: v1.4.3 50 | duckdb_version: v1.4.3 51 | exclude_archs: ${{ github.repository == 'duckdb/duckdb-ui' && 'wasm_mvp;wasm_eh;wasm_threads' || 'linux_arm64;linux_amd64_musl;osx_amd64;windows_amd64_mingw;wasm_mvp;wasm_eh;wasm_threads' }} 52 | extension_name: ui 53 | 54 | duckdb-stable-deploy: 55 | if: ${{ github.repository == 'duckdb/duckdb-ui' && ( startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' ) }} 56 | name: Deploy stable extension binaries 57 | needs: duckdb-stable-build 58 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.4.3 59 | secrets: inherit 60 | with: 61 | extension_name: ui 62 | ci_tools_version: v1.4.3 63 | duckdb_version: v1.4.3 64 | exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' 65 | deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DuckDB UI Extension 2 | 3 | A [DuckDB extension](https://duckdb.org/docs/stable/core_extensions/ui.html) providing a browser-based user interface. 4 | 5 | This repository contains both the extension, implemented in C++, and some packages used by the user interface, implemented in TypeScript. 6 | 7 | While most of the user interface code is not yet publicly available, more of it will added here over time. 8 | 9 | ## Extension 10 | 11 | The primary structure of this repository is based on the [DuckDB extension template](https://github.com/duckdb/extension-template). 12 | 13 | To build the extension: 14 | 15 | ```sh 16 | make 17 | ``` 18 | 19 | This will create the following binaries: 20 | 21 | ```sh 22 | ./build/release/duckdb # DuckDB shell with UI extension 23 | ./build/release/test/unittest # Test runner 24 | ./build/release/extension/ui/ui.duckdb_extension # Loadable extension binary 25 | ``` 26 | 27 | - `duckdb` is the binary for the duckdb shell with the extension code automatically loaded. 28 | - `unittest` is the test runner of duckdb. Again, the extension is already linked into the binary. 29 | - `ui.duckdb_extension` is the loadable binary as it would be distributed. 30 | 31 | To run the extension code, simply start the shell with `./build/release/duckdb`. 32 | 33 | To start the UI from the command line: 34 | 35 | ``` 36 | ./build/release/duckdb -ui 37 | ``` 38 | 39 | To start the UI from SQL: 40 | ``` 41 | call start_ui(); 42 | ``` 43 | 44 | For more usage details, see the [documentation](https://duckdb.org/docs/stable/core_extensions/ui.html). 45 | 46 | ## User Interface Packages 47 | 48 | Some packages used by the browser-based user interface can be found in the `ts` directory. 49 | 50 | See the [README](ts/README.md) in that directory for details. 51 | 52 | ## Architectural Overview 53 | 54 | The extension starts an HTTP server that both serves the UI assets (HTML, JavaScript, etc.) 55 | and handles requests to run SQL and perform other DuckDB operations. 56 | 57 | The server proxies requests for UI assets and fetches them from a remote server. 58 | By default, this is `https://ui.duckdb.org`, but it can be [overridden](https://duckdb.org/docs/stable/core_extensions/ui.html#remote-url). 59 | 60 | The server also exposes a number of HTTP endpoints for performing DuckDB operations. 61 | These include running SQL, interrupting runs, tokenizing SQL text, and receiving events (such as catalog updates). 62 | For details, see the `HttpServer::Run` method in [http_server.cpp](src/http_server.cpp). 63 | 64 | The UI uses the TypeScript package [duckdb-ui-client](ts/pkgs/duckdb-ui-client/package.json) for communicating with the server. 65 | See the [DuckDBUIClient](ts/pkgs/duckdb-ui-client/src/client/classes/DuckDBUIClient.ts) and [DuckDBUIClientConnection](ts/pkgs/duckdb-ui-client/src/client/classes/DuckDBUIClientConnection.ts) classes exposed by this package for details. 66 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/functions/resultReaders.ts: -------------------------------------------------------------------------------- 1 | import { BinaryDeserializer } from '../classes/BinaryDeserializer.js'; 2 | import { ColumnNamesAndTypes } from '../types/ColumnNamesAndTypes.js'; 3 | import { DataChunk } from '../types/DataChunk.js'; 4 | import { 5 | ErrorQueryResult, 6 | QueryResult, 7 | SuccessQueryResult, 8 | } from '../types/QueryResult.js'; 9 | import { TokenizeResult } from '../types/TokenizeResult.js'; 10 | import { TypeIdAndInfo } from '../types/TypeInfo.js'; 11 | import { 12 | readBoolean, 13 | readList, 14 | readString, 15 | readStringList, 16 | readVarInt, 17 | readVarIntList, 18 | } from './basicReaders.js'; 19 | import { readTypeList } from './typeReaders.js'; 20 | import { readVectorList } from './vectorReaders.js'; 21 | 22 | export function readTokenizeResult( 23 | deserializer: BinaryDeserializer, 24 | ): TokenizeResult { 25 | const offsets = deserializer.readProperty(100, readVarIntList); 26 | const types = deserializer.readProperty(101, readVarIntList); 27 | deserializer.expectObjectEnd(); 28 | return { offsets, types }; 29 | } 30 | 31 | export function readColumnNamesAndTypes( 32 | deserializer: BinaryDeserializer, 33 | ): ColumnNamesAndTypes { 34 | const names = deserializer.readProperty(100, readStringList); 35 | const types = deserializer.readProperty(101, readTypeList); 36 | deserializer.expectObjectEnd(); 37 | return { names, types }; 38 | } 39 | 40 | export function readChunk( 41 | deserializer: BinaryDeserializer, 42 | types: TypeIdAndInfo[], 43 | ): DataChunk { 44 | const rowCount = deserializer.readProperty(100, readVarInt); 45 | const vectors = deserializer.readProperty(101, (d) => 46 | readVectorList(d, types), 47 | ); 48 | deserializer.expectObjectEnd(); 49 | return { rowCount, vectors }; 50 | } 51 | 52 | export function readDataChunkList( 53 | deserializer: BinaryDeserializer, 54 | types: TypeIdAndInfo[], 55 | ): DataChunk[] { 56 | return readList(deserializer, (d) => readChunk(d, types)); 57 | } 58 | 59 | export function readSuccessQueryResult( 60 | deserializer: BinaryDeserializer, 61 | ): SuccessQueryResult { 62 | const columnNamesAndTypes = deserializer.readProperty( 63 | 101, 64 | readColumnNamesAndTypes, 65 | ); 66 | const chunks = deserializer.readProperty(102, (d) => 67 | readDataChunkList(d, columnNamesAndTypes.types), 68 | ); 69 | return { success: true, columnNamesAndTypes, chunks }; 70 | } 71 | 72 | export function readErrorQueryResult( 73 | deserializer: BinaryDeserializer, 74 | ): ErrorQueryResult { 75 | const error = deserializer.readProperty(101, readString); 76 | return { success: false, error }; 77 | } 78 | 79 | export function readQueryResult(deserializer: BinaryDeserializer): QueryResult { 80 | const success = deserializer.readProperty(100, readBoolean); 81 | if (success) { 82 | return readSuccessQueryResult(deserializer); 83 | } 84 | return readErrorQueryResult(deserializer); 85 | } 86 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBListValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBMapValue } from '../src'; 3 | import { DuckDBListValue } from '../src/DuckDBListValue'; 4 | import { DuckDBStructValue } from '../src/DuckDBStructValue'; 5 | 6 | suite('DuckDBListValue', () => { 7 | test('should render an empty list to the correct string', () => { 8 | expect(new DuckDBListValue([]).toString()).toStrictEqual('[]'); 9 | }); 10 | test('should render a single element list to the correct string', () => { 11 | expect(new DuckDBListValue([123]).toString()).toStrictEqual('[123]'); 12 | }); 13 | test('should render a multi-element list to the correct string', () => { 14 | expect( 15 | new DuckDBListValue(['abc', null, true, '']).toString(), 16 | ).toStrictEqual(`['abc', NULL, true, '']`); 17 | }); 18 | test('should render a list with nested lists to the correct string', () => { 19 | expect( 20 | new DuckDBListValue([ 21 | new DuckDBListValue([]), 22 | null, 23 | new DuckDBListValue([123, null, 'xyz']), 24 | ]).toString(), 25 | ).toStrictEqual(`[[], NULL, [123, NULL, 'xyz']]`); 26 | }); 27 | test('toJson with complex values', () => { 28 | expect( 29 | new DuckDBListValue([ 30 | new DuckDBMapValue([ 31 | { key: 'foo', value: 123 }, 32 | { key: 'bar', value: 'abc' }, 33 | ]), 34 | null, 35 | new DuckDBMapValue([ 36 | { key: 'foo', value: null }, 37 | { key: 'bar', value: 'xyz' }, 38 | ]), 39 | ]).toJson(), 40 | ).toStrictEqual([ 41 | { "'foo'": 123, "'bar'": 'abc' }, 42 | null, 43 | { "'foo'": null, "'bar'": 'xyz' }, 44 | ]); 45 | }); 46 | 47 | suite('toSql', () => { 48 | test('should render empty list', () => { 49 | expect(new DuckDBListValue([]).toSql()).toStrictEqual('[]'); 50 | }); 51 | 52 | test('should render list with mixed types', () => { 53 | expect( 54 | new DuckDBListValue([123, 'abc', null, true]).toSql(), 55 | ).toStrictEqual("[123, 'abc', NULL, TRUE]"); 56 | }); 57 | 58 | test('should render nested lists', () => { 59 | expect( 60 | new DuckDBListValue([ 61 | new DuckDBListValue([1, 2]), 62 | null, 63 | new DuckDBListValue([3, 4]), 64 | ]).toSql(), 65 | ).toStrictEqual('[[1, 2], NULL, [3, 4]]'); 66 | }); 67 | 68 | test('should render list of structs', () => { 69 | expect( 70 | new DuckDBListValue([ 71 | new DuckDBStructValue([ 72 | { key: 'id', value: 1 }, 73 | { key: 'name', value: 'Alice' }, 74 | ]), 75 | new DuckDBStructValue([ 76 | { key: 'id', value: 2 }, 77 | { key: 'name', value: 'Bob' }, 78 | ]), 79 | ]).toSql(), 80 | ).toStrictEqual("[{'id': 1, 'name': 'Alice'}, {'id': 2, 'name': 'Bob'}]"); 81 | }); 82 | }); 83 | }); 84 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBTimeTZValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBTimeTZValue } from '../src/DuckDBTimeTZValue'; 3 | 4 | suite('DuckDBTimeTZValue', () => { 5 | test('should render a normal time value with a positive offset to the correct string', () => { 6 | expect( 7 | new DuckDBTimeTZValue( 8 | ((12n * 60n + 34n) * 60n + 56n) * 1000000n + 789012n, 9 | (13 * 60 + 24) * 60 + 57, 10 | ).toString(), 11 | ).toStrictEqual('12:34:56.789012+13:24:57'); 12 | }); 13 | test('should render a normal time value with millisecond precision with an offset in minutes to the correct string', () => { 14 | expect( 15 | new DuckDBTimeTZValue( 16 | ((12n * 60n + 34n) * 60n + 56n) * 1000000n + 789000n, 17 | (13 * 60 + 24) * 60, 18 | ).toString(), 19 | ).toStrictEqual('12:34:56.789+13:24'); 20 | }); 21 | test('should render a normal time value with second precision with an offset in hours to the correct string', () => { 22 | expect( 23 | new DuckDBTimeTZValue( 24 | ((12n * 60n + 34n) * 60n + 56n) * 1000000n, 25 | (13 * 60 + 0) * 60, 26 | ).toString(), 27 | ).toStrictEqual('12:34:56+13'); 28 | }); 29 | test('should render a zero time value with a zero offset to the correct string', () => { 30 | expect(new DuckDBTimeTZValue(0n, 0).toString()).toStrictEqual( 31 | '00:00:00+00', 32 | ); 33 | }); 34 | test('should render the max value to the correct string', () => { 35 | expect( 36 | new DuckDBTimeTZValue( 37 | ((24n * 60n + 0n) * 60n + 0n) * 1000000n, 38 | -((15 * 60 + 59) * 60 + 59), 39 | ).toString(), 40 | ).toStrictEqual('24:00:00-15:59:59'); 41 | }); 42 | test('should render the min value to the correct string', () => { 43 | expect( 44 | new DuckDBTimeTZValue(0n, (15 * 60 + 59) * 60 + 59).toString(), 45 | ).toStrictEqual('00:00:00+15:59:59'); 46 | }); 47 | test('should construct the correct value from bits', () => { 48 | expect(DuckDBTimeTZValue.fromBits(0n).toString()).toStrictEqual( 49 | '00:00:00+15:59:59', 50 | ); 51 | }); 52 | test('should construct the correct value from bits', () => { 53 | expect( 54 | DuckDBTimeTZValue.fromBits( 55 | (BigInt.asUintN(40, ((24n * 60n + 0n) * 60n + 0n) * 1000000n) << 24n) | 56 | BigInt.asUintN(24, (31n * 60n + 59n) * 60n + 58n), 57 | ).toString(), 58 | ).toStrictEqual('24:00:00-15:59:59'); 59 | }); 60 | 61 | suite('toSql', () => { 62 | test('should render time with timezone to SQL', () => { 63 | const micros = BigInt(12 * 3600 + 30 * 60 + 45) * 1000000n; 64 | const offset = 3600; 65 | expect(new DuckDBTimeTZValue(micros, offset).toSql()).toStrictEqual( 66 | "TIMETZ '12:30:45+01'", 67 | ); 68 | }); 69 | 70 | test('should render midnight UTC to SQL', () => { 71 | expect(new DuckDBTimeTZValue(0n, 0).toSql()).toStrictEqual( 72 | "TIMETZ '00:00:00+00'", 73 | ); 74 | }); 75 | }); 76 | }); 77 | -------------------------------------------------------------------------------- /ts/README.md: -------------------------------------------------------------------------------- 1 | # TypeScript Workspace 2 | 3 | ## Structure 4 | 5 | This directory is a [pnpm workspace](https://pnpm.io/workspaces). Use the [pnpm](https://pnpm.io/) package manager, not npm or yarn. 6 | 7 | One (recommended) way to install pnpm is using [corepack](https://pnpm.io/installation#using-corepack). 8 | 9 | ## Build 10 | 11 | Run `pnpm install` (or just `pnpm i`) in a package directory to install dependencies and build. Note that this will also build dependent packages in this workspace. This builds src files, but not test files. 12 | 13 | Run `pnpm build` to just run the build. This will not build dependencies. It will build both src and test files. To build just src or just test, use `pnpm build:src` or `pnpm build:test`. 14 | 15 | Run `pnpm build:watch` in a package to rebuild (both src and test files) when source files are changed. 16 | 17 | Run `pnpm check` in a package to check formatting and linting rules. To just check formatting, run `pnpm format:check`. To correct formatting, run `pnpm format:write`. To just check linting rules, run `pnpm lint`. 18 | 19 | Run `pnpm clean` in that package to remove built output files for that package. 20 | 21 | Run `pnpm build` at the root of the workspace to build all packages (both src and test files). 22 | 23 | Run `pnpm build:watch` at the root can be used to rebuild (only) relevant packages when source files are changed. 24 | 25 | Run `pnpm check` at the root of the workspace to check formatting and linting rules all packages. 26 | 27 | ## Test 28 | 29 | Run `pnpm test` in a package directory to run its tests. 30 | 31 | Run `pnpm test:watch` in a package directory to run its tests and rerun when source files change. 32 | 33 | Tests use [vitest](https://vitest.dev/), either in Node or in [Browser Mode](https://vitest.dev/guide/browser.html) (using Chrome), depending on the package. 34 | 35 | Run `pnpm test` at the root of the workspace to test all packages. 36 | 37 | ## Create 38 | 39 | To create a new package, add a directory under `packages`. 40 | 41 | Add a `package.json` file following the conventions of other packages. 42 | 43 | The `package.json` should have `preinstall`, `build`, `clean`, and `test` scripts, as well as 'check', 'format', and 'lint' scripts. See existing packages for details. 44 | It should have a `name`, `version`, and `description`, set `"type": "module"`, and set `main`, `module`, and `types` appropriately. 45 | 46 | Production source code should go in a `src` subdirectory. 47 | Put a `tsconfig.json` in this directory that extends `tsconfig.library.json` and sets the `outDir` to `../out`. 48 | 49 | Test source code should got in a `test` subdirectory. 50 | Put a `tsconfig.json` in this directory that extends `tsconfig.test.json` and references `../src`. 51 | 52 | For browser-based tests, create a `vite.config.js` file, and enable `browser` mode, set the `headless` option to `true`, and set the `type` to `chrome`. 53 | Note that `crossOriginIsolated` can be enabled by setting server headers. See example in `wasm-extension`. 54 | 55 | Add references to both the `src` and `test` directories of your new package to the root `tsconfig.json` of the workspace. 56 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBTimestampMicrosecondsValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBTimestampMicrosecondsValue } from '../src/DuckDBTimestampMicrosecondsValue'; 3 | 4 | suite('DuckDBTimestampMicrosecondsValue', () => { 5 | test('should render a normal timestamp value to the correct string', () => { 6 | expect( 7 | new DuckDBTimestampMicrosecondsValue(1612325106007800n).toString(), 8 | ).toStrictEqual('2021-02-03 04:05:06.0078'); 9 | }); 10 | test('should render a zero timestamp value to the correct string', () => { 11 | expect(new DuckDBTimestampMicrosecondsValue(0n).toString()).toStrictEqual( 12 | '1970-01-01 00:00:00', 13 | ); 14 | }); 15 | test('should render a negative timestamp value to the correct string', () => { 16 | expect(new DuckDBTimestampMicrosecondsValue(-7n).toString()).toStrictEqual( 17 | '1969-12-31 23:59:59.999993', 18 | ); 19 | }); 20 | test('should render a large positive timestamp value to the correct string', () => { 21 | expect( 22 | new DuckDBTimestampMicrosecondsValue(2353318271999999000n).toString(), 23 | ).toStrictEqual('76543-09-08 23:59:59.999'); 24 | }); 25 | test('should render a large negative (AD) timestamp value to the correct string', () => { 26 | expect( 27 | new DuckDBTimestampMicrosecondsValue(-58261244276543211n).toString(), 28 | ).toStrictEqual('0123-10-11 01:02:03.456789'); 29 | }); 30 | test('should render a large negative (BC) timestamp value to the correct string', () => { 31 | expect( 32 | new DuckDBTimestampMicrosecondsValue(-65992661876543211n).toString(), 33 | ).toStrictEqual('0123-10-11 (BC) 01:02:03.456789'); 34 | }); 35 | test('should render the max timestamp value to the correct string', () => { 36 | expect( 37 | new DuckDBTimestampMicrosecondsValue(9223372036854775806n).toString(), 38 | ).toStrictEqual('294247-01-10 04:00:54.775806'); 39 | }); 40 | test('should render the min timestamp value to the correct string', () => { 41 | expect( 42 | new DuckDBTimestampMicrosecondsValue(-9223372022400000000n).toString(), 43 | ).toStrictEqual('290309-12-22 (BC) 00:00:00'); 44 | }); 45 | test('should render the positive infinity timestamp value to the correct string', () => { 46 | expect( 47 | new DuckDBTimestampMicrosecondsValue(9223372036854775807n).toString(), 48 | ).toStrictEqual('infinity'); 49 | }); 50 | test('should render the negative infinity timestamp value to the correct string', () => { 51 | expect( 52 | new DuckDBTimestampMicrosecondsValue(-9223372036854775807n).toString(), 53 | ).toStrictEqual('-infinity'); 54 | }); 55 | 56 | suite('toSql', () => { 57 | test('should render timestamp to SQL', () => { 58 | const timestamp = new DuckDBTimestampMicrosecondsValue( 59 | BigInt(1697212800) * 1000000n, 60 | ); 61 | expect(timestamp.toSql()).toMatch(/^TIMESTAMP '.+'$/); 62 | }); 63 | 64 | test('should render epoch to SQL', () => { 65 | expect(new DuckDBTimestampMicrosecondsValue(0n).toSql()).toStrictEqual( 66 | "TIMESTAMP '1970-01-01 00:00:00'", 67 | ); 68 | }); 69 | }); 70 | }); 71 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/conversion/duckDBValueToSql.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { duckDBValueToSql } from '../../src/conversion/duckDBValueToSql'; 3 | 4 | // tests primitives 5 | suite('duckDBValueToSql', () => { 6 | suite('null', () => { 7 | test('should convert null to NULL', () => { 8 | expect(duckDBValueToSql(null)).toStrictEqual('NULL'); 9 | }); 10 | }); 11 | 12 | suite('string', () => { 13 | test('should wrap simple string in single quotes', () => { 14 | expect(duckDBValueToSql('hello')).toStrictEqual("'hello'"); 15 | }); 16 | 17 | test('should escape single quotes by doubling them', () => { 18 | expect(duckDBValueToSql("it's")).toStrictEqual("'it''s'"); 19 | }); 20 | 21 | test('should handle multiple single quotes', () => { 22 | expect(duckDBValueToSql("don't say 'no'")).toStrictEqual( 23 | "'don''t say ''no'''", 24 | ); 25 | }); 26 | 27 | test('should wrap empty string in single quotes', () => { 28 | expect(duckDBValueToSql('')).toStrictEqual("''"); 29 | }); 30 | 31 | test('should handle strings with special characters', () => { 32 | expect(duckDBValueToSql('hello\nworld')).toStrictEqual("'hello\nworld'"); 33 | }); 34 | }); 35 | 36 | suite('number', () => { 37 | test('should convert integer to string', () => { 38 | expect(duckDBValueToSql(123)).toStrictEqual('123'); 39 | }); 40 | 41 | test('should convert float to string', () => { 42 | expect(duckDBValueToSql(123.456)).toStrictEqual('123.456'); 43 | }); 44 | 45 | test('should convert negative number to string', () => { 46 | expect(duckDBValueToSql(-42)).toStrictEqual('-42'); 47 | }); 48 | 49 | test('should convert zero to string', () => { 50 | expect(duckDBValueToSql(0)).toStrictEqual('0'); 51 | }); 52 | 53 | test('should handle scientific notation', () => { 54 | expect(duckDBValueToSql(1.23e5)).toStrictEqual('123000'); 55 | }); 56 | 57 | test('should handle very small numbers', () => { 58 | expect(duckDBValueToSql(0.00001)).toStrictEqual('0.00001'); 59 | }); 60 | }); 61 | 62 | suite('boolean', () => { 63 | test('should convert true to TRUE', () => { 64 | expect(duckDBValueToSql(true)).toStrictEqual('TRUE'); 65 | }); 66 | 67 | test('should convert false to FALSE', () => { 68 | expect(duckDBValueToSql(false)).toStrictEqual('FALSE'); 69 | }); 70 | }); 71 | 72 | suite('bigint', () => { 73 | test('should convert small bigint to string', () => { 74 | expect(duckDBValueToSql(123n)).toStrictEqual('123'); 75 | }); 76 | 77 | test('should convert large bigint to string', () => { 78 | expect(duckDBValueToSql(9007199254740991n)).toStrictEqual( 79 | '9007199254740991', 80 | ); 81 | }); 82 | 83 | test('should convert very large bigint beyond Number.MAX_SAFE_INTEGER', () => { 84 | expect(duckDBValueToSql(12345678901234567890n)).toStrictEqual( 85 | '12345678901234567890', 86 | ); 87 | }); 88 | 89 | test('should convert negative bigint to string', () => { 90 | expect(duckDBValueToSql(-999n)).toStrictEqual('-999'); 91 | }); 92 | }); 93 | }); 94 | -------------------------------------------------------------------------------- /src/include/http_server.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #define CPPHTTPLIB_OPENSSL_SUPPORT 7 | #include "httplib.hpp" 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "event_dispatcher.hpp" 14 | #include "watcher.hpp" 15 | 16 | namespace httplib = duckdb_httplib_openssl; 17 | 18 | namespace duckdb { 19 | class HTTPParams; 20 | class MemoryStream; 21 | 22 | namespace ui { 23 | 24 | class HttpServer { 25 | 26 | public: 27 | HttpServer(shared_ptr _ddb_instance) 28 | : ddb_instance(_ddb_instance) {} 29 | 30 | static HttpServer *GetInstance(ClientContext &); 31 | static void UpdateDatabaseInstanceIfRunning(shared_ptr); 32 | static bool IsRunningOnMachine(ClientContext &); 33 | static bool Started(); 34 | static void StopInstance(); 35 | 36 | static const HttpServer &Start(ClientContext &, bool *was_started = nullptr); 37 | static bool Stop(); 38 | 39 | std::string LocalUrl() const; 40 | 41 | private: 42 | friend class Watcher; 43 | 44 | // Lifecycle 45 | void DoStart(const uint16_t local_port, const std::string &remote_url, 46 | unique_ptr); 47 | void DoStop(); 48 | void Run(); 49 | void UpdateDatabaseInstance(shared_ptr context_db); 50 | 51 | // Http handlers 52 | void HandleGetInfo(const httplib::Request &req, httplib::Response &res); 53 | void HandleGetLocalEvents(const httplib::Request &req, 54 | httplib::Response &res); 55 | void HandleGetLocalToken(const httplib::Request &req, httplib::Response &res); 56 | void HandleGet(const httplib::Request &req, httplib::Response &res); 57 | void HandleInterrupt(const httplib::Request &req, httplib::Response &res); 58 | void DoHandleRun(const httplib::Request &req, httplib::Response &res, 59 | const httplib::ContentReader &content_reader); 60 | void HandleRun(const httplib::Request &req, httplib::Response &res, 61 | const httplib::ContentReader &content_reader); 62 | void HandleTokenize(const httplib::Request &req, httplib::Response &res, 63 | const httplib::ContentReader &content_reader); 64 | std::string ReadContent(const httplib::ContentReader &content_reader); 65 | 66 | // Http responses 67 | void SetResponseContent(httplib::Response &res, const MemoryStream &content); 68 | void SetResponseEmptyResult(httplib::Response &res); 69 | void SetResponseErrorResult(httplib::Response &res, const std::string &error); 70 | 71 | // Misc 72 | shared_ptr LockDatabaseInstance(); 73 | void InitClientFromParams(httplib::Client &); 74 | 75 | static void CopyAndSlice(duckdb::DataChunk &source, duckdb::DataChunk &target, idx_t row_count); 76 | 77 | uint16_t local_port; 78 | std::string local_url; 79 | std::string remote_url; 80 | weak_ptr ddb_instance; 81 | std::string user_agent; 82 | httplib::Server server; 83 | unique_ptr main_thread; 84 | unique_ptr event_dispatcher; 85 | unique_ptr watcher; 86 | unique_ptr http_params; 87 | 88 | static unique_ptr server_instance; 89 | }; 90 | ; 91 | 92 | } // namespace ui 93 | } // namespace duckdb 94 | -------------------------------------------------------------------------------- /scripts/extension-upload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Extension upload script 4 | 5 | # Usage: ./extension-upload.sh 6 | # : Name of the extension 7 | # : Version (commit / version tag) of the extension 8 | # : Version (commit / version tag) of DuckDB 9 | # : Architecture target of the extension binary 10 | # : S3 bucket to upload to 11 | # : Set this as the latest version ("true" / "false", default: "false") 12 | # : Set this as a versioned version that will prevent its deletion 13 | 14 | set -e 15 | 16 | if [[ $4 == wasm* ]]; then 17 | ext="/tmp/extension/$1.duckdb_extension.wasm" 18 | else 19 | ext="/tmp/extension/$1.duckdb_extension" 20 | fi 21 | 22 | echo $ext 23 | 24 | script_dir="$(dirname "$(readlink -f "$0")")" 25 | 26 | # calculate SHA256 hash of extension binary 27 | cat $ext > $ext.append 28 | 29 | if [[ $4 == wasm* ]]; then 30 | # 0 for custom section 31 | # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) 32 | # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] 33 | echo -n -e '\x00' >> $ext.append 34 | echo -n -e '\x93\x02' >> $ext.append 35 | # 10 in hex = 16 in decimal, lenght of name, 1 byte 36 | echo -n -e '\x10' >> $ext.append 37 | echo -n -e 'duckdb_signature' >> $ext.append 38 | # the name of the WebAssembly custom section, 16 bytes 39 | # 100 in hex, 256 in decimal 40 | # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], 41 | # for a grand total of 2 bytes 42 | echo -n -e '\x80\x02' >> $ext.append 43 | fi 44 | 45 | # (Optionally) Sign binary 46 | if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then 47 | echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem 48 | $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash 49 | openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign 50 | rm -f private.pem 51 | fi 52 | 53 | # Signature is always there, potentially defaulting to 256 zeros 54 | truncate -s 256 $ext.sign 55 | 56 | # append signature to extension binary 57 | cat $ext.sign >> $ext.append 58 | 59 | # compress extension binary 60 | if [[ $4 == wasm_* ]]; then 61 | brotli < $ext.append > "$ext.compressed" 62 | else 63 | gzip < $ext.append > "$ext.compressed" 64 | fi 65 | 66 | set -e 67 | 68 | # Abort if AWS key is not set 69 | if [ -z "$AWS_ACCESS_KEY_ID" ]; then 70 | echo "No AWS key found, skipping.." 71 | exit 0 72 | fi 73 | 74 | # upload versioned version 75 | if [[ $7 = 'true' ]]; then 76 | if [[ $4 == wasm* ]]; then 77 | aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" 78 | else 79 | aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read 80 | fi 81 | fi 82 | 83 | # upload to latest version 84 | if [[ $6 = 'true' ]]; then 85 | if [[ $4 == wasm* ]]; then 86 | aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" 87 | else 88 | aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read 89 | fi 90 | fi 91 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/serialization/classes/BinaryDeserializer.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { BinaryDeserializer } from '../../../src/serialization/classes/BinaryDeserializer'; 3 | import { BinaryStreamReader } from '../../../src/serialization/classes/BinaryStreamReader'; 4 | import { 5 | readString, 6 | readUint8, 7 | } from '../../../src/serialization/functions/basicReaders'; 8 | import { makeBuffer } from '../../helpers/makeBuffer'; 9 | 10 | suite('BinaryDeserializer', () => { 11 | test('read uint8', () => { 12 | const deserializer = new BinaryDeserializer( 13 | new BinaryStreamReader(makeBuffer([17, 42])), 14 | ); 15 | expect(deserializer.readUint8()).toBe(17); 16 | expect(deserializer.readUint8()).toBe(42); 17 | }); 18 | test('read varint', () => { 19 | const deserializer = new BinaryDeserializer( 20 | new BinaryStreamReader(makeBuffer([0x81, 0x82, 0x03])), 21 | ); 22 | expect(deserializer.readVarInt()).toBe((3 << 14) | (2 << 7) | 1); 23 | }); 24 | test('read nullable', () => { 25 | const deserializer = new BinaryDeserializer( 26 | new BinaryStreamReader(makeBuffer([0, 1, 17])), 27 | ); 28 | expect(deserializer.readNullable(readUint8)).toBe(null); 29 | expect(deserializer.readNullable(readUint8)).toBe(17); 30 | }); 31 | test('read data', () => { 32 | const deserializer = new BinaryDeserializer( 33 | new BinaryStreamReader(makeBuffer([3, 0xa, 0xb, 0xc])), 34 | ); 35 | const dv = deserializer.readData(); 36 | expect(dv.byteLength).toBe(3); 37 | expect(dv.getUint8(0)).toBe(0xa); 38 | expect(dv.getUint8(1)).toBe(0xb); 39 | expect(dv.getUint8(2)).toBe(0xc); 40 | }); 41 | test('read string', () => { 42 | const deserializer = new BinaryDeserializer( 43 | new BinaryStreamReader(makeBuffer([4, 0x64, 0x75, 0x63, 0x6b])), 44 | ); 45 | expect(deserializer.readString()).toBe('duck'); 46 | }); 47 | test('read list (of string)', () => { 48 | const deserializer = new BinaryDeserializer( 49 | new BinaryStreamReader( 50 | makeBuffer([ 51 | 3, 4, 0x77, 0x61, 0x6c, 0x6b, 4, 0x73, 0x77, 0x69, 0x6d, 3, 0x66, 52 | 0x6c, 0x79, 53 | ]), 54 | ), 55 | ); 56 | expect(deserializer.readList(readString)).toEqual(['walk', 'swim', 'fly']); 57 | }); 58 | test('read pair', () => { 59 | const deserializer = new BinaryDeserializer( 60 | new BinaryStreamReader( 61 | makeBuffer([0, 0, 4, 0x64, 0x75, 0x63, 0x6b, 1, 0, 42, 0xff, 0xff]), 62 | ), 63 | ); 64 | expect(deserializer.readPair(readString, readUint8)).toEqual(['duck', 42]); 65 | }); 66 | test('read property', () => { 67 | const deserializer = new BinaryDeserializer( 68 | new BinaryStreamReader(makeBuffer([100, 0, 4, 0x64, 0x75, 0x63, 0x6b])), 69 | ); 70 | expect(deserializer.readProperty(100, readString)).toEqual('duck'); 71 | }); 72 | test('read property (not present)', () => { 73 | const deserializer = new BinaryDeserializer( 74 | new BinaryStreamReader(makeBuffer([100, 0, 4, 0x64, 0x75, 0x63, 0x6b])), 75 | ); 76 | expect(() => deserializer.readProperty(101, readString)).toThrowError( 77 | 'Expected field id 101 but got 100 (offset=0)', 78 | ); 79 | }); 80 | test('read property with default', () => { 81 | const deserializer = new BinaryDeserializer( 82 | new BinaryStreamReader(makeBuffer([101, 0, 42])), 83 | ); 84 | expect(deserializer.readPropertyWithDefault(100, readUint8, 17)).toBe(17); 85 | expect(deserializer.readPropertyWithDefault(101, readUint8, 17)).toBe(42); 86 | }); 87 | }); 88 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/DuckDBBitValue.ts: -------------------------------------------------------------------------------- 1 | import { Json } from './Json.js'; 2 | import { SpecialDuckDBValue } from './SpecialDuckDBValue.js'; 3 | 4 | export class DuckDBBitValue extends SpecialDuckDBValue { 5 | public readonly data: Uint8Array; 6 | 7 | constructor(data: Uint8Array) { 8 | super(); 9 | this.data = data; 10 | } 11 | 12 | public padding(): number { 13 | return this.data[0]; 14 | } 15 | 16 | public get length(): number { 17 | return (this.data.length - 1) * 8 - this.padding(); 18 | } 19 | 20 | public getBool(index: number): boolean { 21 | const offset = index + this.padding(); 22 | const dataIndex = Math.floor(offset / 8) + 1; 23 | const byte = this.data[dataIndex] >> (7 - (offset % 8)); 24 | return (byte & 1) !== 0; 25 | } 26 | 27 | public toBools(): boolean[] { 28 | const bools: boolean[] = []; 29 | const length = this.length; 30 | for (let i = 0; i < length; i++) { 31 | bools.push(this.getBool(i)); 32 | } 33 | return bools; 34 | } 35 | 36 | public getBit(index: number): 0 | 1 { 37 | return this.getBool(index) ? 1 : 0; 38 | } 39 | 40 | public toBits(): number[] { 41 | const bits: number[] = []; 42 | const length = this.length; 43 | for (let i = 0; i < length; i++) { 44 | bits.push(this.getBit(i)); 45 | } 46 | return bits; 47 | } 48 | 49 | public toDuckDBString(): string { 50 | const length = this.length; 51 | const chars = Array.from({ length }); 52 | for (let i = 0; i < length; i++) { 53 | chars[i] = this.getBool(i) ? '1' : '0'; 54 | } 55 | return chars.join(''); 56 | } 57 | 58 | public toSql(): string { 59 | return `'${this.toDuckDBString()}'::BITSTRING`; 60 | } 61 | 62 | public toJson(): Json { 63 | return this.toDuckDBString(); 64 | } 65 | 66 | public static fromString(str: string, on: string = '1'): DuckDBBitValue { 67 | return DuckDBBitValue.fromLengthAndPredicate( 68 | str.length, 69 | (i) => str[i] === on, 70 | ); 71 | } 72 | 73 | public static fromBits( 74 | bits: readonly number[], 75 | on: number = 1, 76 | ): DuckDBBitValue { 77 | return DuckDBBitValue.fromLengthAndPredicate( 78 | bits.length, 79 | (i) => bits[i] === on, 80 | ); 81 | } 82 | 83 | public static fromBools(bools: readonly boolean[]): DuckDBBitValue { 84 | return DuckDBBitValue.fromLengthAndPredicate(bools.length, (i) => bools[i]); 85 | } 86 | 87 | public static fromLengthAndPredicate( 88 | length: number, 89 | predicate: (index: number) => boolean, 90 | ): DuckDBBitValue { 91 | const byteCount = Math.ceil(length / 8) + 1; 92 | const paddingBitCount = (8 - (length % 8)) % 8; 93 | 94 | const data = new Uint8Array(byteCount); 95 | let byteIndex = 0; 96 | 97 | // first byte contains count of padding bits 98 | data[byteIndex++] = paddingBitCount; 99 | 100 | let byte = 0; 101 | let byteBit = 0; 102 | 103 | // padding consists of 1s in MSB of second byte 104 | while (byteBit < paddingBitCount) { 105 | byte <<= 1; 106 | byte |= 1; 107 | byteBit++; 108 | } 109 | 110 | let bitIndex = 0; 111 | 112 | while (byteIndex < byteCount) { 113 | while (byteBit < 8) { 114 | byte <<= 1; 115 | if (predicate(bitIndex++)) { 116 | byte |= 1; 117 | } 118 | byteBit++; 119 | } 120 | data[byteIndex++] = byte; 121 | byte = 0; 122 | byteBit = 0; 123 | } 124 | 125 | return new DuckDBBitValue(data); 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/classes/BinaryDeserializer.ts: -------------------------------------------------------------------------------- 1 | import { BinaryStreamReader } from './BinaryStreamReader.js'; 2 | 3 | export type Reader = (deserializer: BinaryDeserializer) => T; 4 | export type ListReader = ( 5 | deserializer: BinaryDeserializer, 6 | index: number, 7 | ) => T; 8 | 9 | const decoder = new TextDecoder(); 10 | 11 | /** 12 | * An implementation of a subset of DuckDB's BinaryDeserializer. 13 | * 14 | * See: 15 | * - https://github.com/duckdb/duckdb/blob/main/src/include/duckdb/common/serializer/binary_deserializer.hpp 16 | * - https://github.com/duckdb/duckdb/blob/main/src/common/serializer/binary_deserializer.cpp 17 | */ 18 | export class BinaryDeserializer { 19 | private reader: BinaryStreamReader; 20 | 21 | public constructor(reader: BinaryStreamReader) { 22 | this.reader = reader; 23 | } 24 | 25 | private peekFieldId() { 26 | return this.reader.peekUint16(true); 27 | } 28 | 29 | private consumeFieldId() { 30 | this.reader.consume(2); 31 | } 32 | 33 | private checkFieldId(possibleFieldId: number) { 34 | const fieldId = this.peekFieldId(); 35 | if (fieldId === possibleFieldId) { 36 | this.consumeFieldId(); 37 | return true; 38 | } 39 | return false; 40 | } 41 | 42 | private expectFieldId(expectedFieldId: number) { 43 | const fieldId = this.peekFieldId(); 44 | if (fieldId === expectedFieldId) { 45 | this.consumeFieldId(); 46 | } else { 47 | throw new Error( 48 | `Expected field id ${expectedFieldId} but got ${fieldId} (offset=${this.reader.getOffset()})`, 49 | ); 50 | } 51 | } 52 | 53 | public expectObjectEnd() { 54 | this.expectFieldId(0xffff); 55 | } 56 | 57 | public throwUnsupported() { 58 | throw new Error(`unsupported type, offset=${this.reader.getOffset()}`); 59 | } 60 | 61 | public readUint8() { 62 | return this.reader.readUint8(); 63 | } 64 | 65 | public readVarInt() { 66 | let result = 0; 67 | let byte = 0; 68 | let shift = 0; 69 | do { 70 | byte = this.reader.readUint8(); 71 | result |= (byte & 0x7f) << shift; 72 | shift += 7; 73 | } while (byte & 0x80); 74 | return result; 75 | } 76 | 77 | public readNullable(reader: Reader) { 78 | const present = this.readUint8(); 79 | if (present) { 80 | return reader(this); 81 | } 82 | return null; 83 | } 84 | 85 | public readData() { 86 | const length = this.readVarInt(); 87 | return this.reader.readData(length); 88 | } 89 | 90 | public readString() { 91 | const length = this.readVarInt(); 92 | const dv = this.reader.readData(length); 93 | return decoder.decode(dv); 94 | } 95 | 96 | public readList(reader: ListReader) { 97 | const count = this.readVarInt(); 98 | const items: T[] = []; 99 | for (let i = 0; i < count; i++) { 100 | items.push(reader(this, i)); 101 | } 102 | return items; 103 | } 104 | 105 | public readPair( 106 | firstReader: Reader, 107 | secondReader: Reader, 108 | ): [T, U] { 109 | const first = this.readProperty(0, firstReader); 110 | const second = this.readProperty(1, secondReader); 111 | this.expectObjectEnd(); 112 | return [first, second]; 113 | } 114 | 115 | public readProperty(expectedFieldId: number, reader: Reader) { 116 | this.expectFieldId(expectedFieldId); 117 | return reader(this); 118 | } 119 | 120 | public readPropertyWithDefault( 121 | possibleFieldId: number, 122 | reader: Reader, 123 | defaultValue: T, 124 | ): T { 125 | if (this.checkFieldId(possibleFieldId)) { 126 | return reader(this); 127 | } 128 | return defaultValue; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/conversion/getBigNumFromBytes.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { getBigNumFromBytes } from '../../src/conversion/getBigNumFromBytes'; 3 | 4 | suite('getBigNumFromBytes', () => { 5 | test('should return correct value for bignum representation of 0', () => { 6 | expect( 7 | getBigNumFromBytes(new Uint8Array([0x80, 0x00, 0x01, 0x00])), 8 | ).toEqual(0n); 9 | }); 10 | test('should return correct value for bignum representation of 1', () => { 11 | expect( 12 | getBigNumFromBytes(new Uint8Array([0x80, 0x00, 0x01, 0x01])), 13 | ).toEqual(1n); 14 | }); 15 | test('should return correct value for bignum representation of -1', () => { 16 | expect( 17 | getBigNumFromBytes(new Uint8Array([0x7f, 0xff, 0xfe, 0xfe])), 18 | ).toEqual(-1n); 19 | }); 20 | test('should return correct value for max bignum', () => { 21 | // max BIGNUM = max IEEE double = 2^1023 * (1 + (1 − 2^−52)) ~= 1.7976931348623157 * 10^308 22 | // Note that the storage format supports much larger than this, but DuckDB specifies this max to support conversion to/from DOUBLE. 23 | expect( 24 | getBigNumFromBytes( 25 | // prettier-ignore 26 | new Uint8Array([0x80, 0x00, 0x80, 27 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 28 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 29 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 30 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 31 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 32 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 33 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 34 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 35 | ]), 36 | ), 37 | ).toEqual( 38 | 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368n, 39 | ); 40 | }); 41 | test('should return correct value for min bignum', () => { 42 | // min BIGNUM = -max BIGNUM 43 | expect( 44 | getBigNumFromBytes( 45 | // prettier-ignore 46 | new Uint8Array([0x7F, 0xFF, 0x7F, 47 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 48 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 49 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 50 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 51 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 52 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 53 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 54 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 55 | ]), 56 | ), 57 | ).toEqual( 58 | -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368n, 59 | ); 60 | }); 61 | }); 62 | -------------------------------------------------------------------------------- /src/watcher.cpp: -------------------------------------------------------------------------------- 1 | #include "watcher.hpp" 2 | 3 | #include 4 | 5 | #include "utils/helpers.hpp" 6 | #include "utils/md_helpers.hpp" 7 | #include "http_server.hpp" 8 | #include "settings.hpp" 9 | 10 | namespace duckdb { 11 | namespace ui { 12 | 13 | Watcher::Watcher(HttpServer &_server) 14 | : should_run(false), server(_server), watched_database(nullptr) {} 15 | 16 | bool WasCatalogUpdated(DatabaseInstance &db, Connection &connection, 17 | CatalogState &last_state) { 18 | bool has_change = false; 19 | auto &context = *connection.context; 20 | connection.BeginTransaction(); 21 | 22 | const auto &databases = db.GetDatabaseManager().GetDatabases(context); 23 | std::set db_oids; 24 | 25 | // Check currently attached databases 26 | for (const auto &db_ref : databases) { 27 | #if DUCKDB_VERSION_AT_MOST(1, 3, 2) 28 | auto &db_instance = db_ref.get(); 29 | #else 30 | auto &db_instance = *db_ref; 31 | #endif 32 | if (db_instance.IsTemporary()) { 33 | continue; // ignore temp databases 34 | } 35 | 36 | db_oids.insert(db_instance.oid); 37 | auto &catalog = db_instance.GetCatalog(); 38 | auto current_version = catalog.GetCatalogVersion(context); 39 | auto last_version_it = last_state.db_to_catalog_version.find(db_instance.oid); 40 | if (last_version_it == last_state.db_to_catalog_version.end() // first time 41 | || !(last_version_it->second == current_version)) { // updated 42 | has_change = true; 43 | last_state.db_to_catalog_version[db_instance.oid] = current_version; 44 | } 45 | } 46 | 47 | // Now check if any databases have been detached 48 | for (auto it = last_state.db_to_catalog_version.begin(); 49 | it != last_state.db_to_catalog_version.end();) { 50 | if (db_oids.find(it->first) == db_oids.end()) { 51 | has_change = true; 52 | it = last_state.db_to_catalog_version.erase(it); 53 | } else { 54 | ++it; 55 | } 56 | } 57 | 58 | connection.Rollback(); 59 | return has_change; 60 | } 61 | 62 | void Watcher::Watch() { 63 | CatalogState last_state; 64 | bool is_md_connected = false; 65 | while (should_run) { 66 | auto db = server.LockDatabaseInstance(); 67 | if (!db) { 68 | break; // DB went away, nothing to watch 69 | } 70 | 71 | if (watched_database == nullptr) { 72 | watched_database = db.get(); 73 | } else if (watched_database != db.get()) { 74 | break; // DB changed, stop watching, will be restarted 75 | } 76 | 77 | duckdb::Connection con{*db}; 78 | auto polling_interval = GetPollingInterval(*con.context); 79 | if (polling_interval == 0) { 80 | return; // Disable watcher 81 | } 82 | 83 | try { 84 | if (WasCatalogUpdated(*db, con, last_state)) { 85 | server.event_dispatcher->SendCatalogChangedEvent(); 86 | } 87 | 88 | if (!is_md_connected && IsMDConnected(con)) { 89 | is_md_connected = true; 90 | server.event_dispatcher->SendConnectedEvent(GetMDToken(con)); 91 | } 92 | } catch (std::exception &ex) { 93 | // Do not crash with uncaught exception, but quit. 94 | std::cerr << "Error in watcher: " << ex.what() << std::endl; 95 | std::cerr << "Will now terminate." << std::endl; 96 | return; 97 | } 98 | 99 | { 100 | std::unique_lock lock(mutex); 101 | cv.wait_for(lock, std::chrono::milliseconds(polling_interval)); 102 | } 103 | } 104 | } 105 | 106 | void Watcher::Start() { 107 | { 108 | std::lock_guard guard(mutex); 109 | should_run = true; 110 | } 111 | 112 | if (!thread) { 113 | thread = make_uniq(&Watcher::Watch, this); 114 | } 115 | } 116 | 117 | void Watcher::Stop() { 118 | if (!thread) { 119 | return; 120 | } 121 | 122 | { 123 | std::lock_guard guard(mutex); 124 | should_run = false; 125 | } 126 | cv.notify_all(); 127 | thread->join(); 128 | thread.reset(); 129 | } 130 | 131 | } // namespace ui 132 | } // namespace duckdb 133 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-reader/src/DuckDBData.ts: -------------------------------------------------------------------------------- 1 | import { DuckDBType } from '@duckdb/data-types'; 2 | import { DuckDBValue } from '@duckdb/data-values'; 3 | import { DuckDBRow } from './DuckDBRow.js'; 4 | 5 | /** 6 | * A two-dimensional table of data along with column metadata. 7 | * 8 | * May represent either a partial or full result set, or a batch of rows read from a result stream. 9 | * */ 10 | export abstract class DuckDBData { 11 | /** 12 | * Number of columns. 13 | * 14 | * May be zero until the first part of the result is read. Will not change after the initial read. 15 | */ 16 | abstract get columnCount(): number; 17 | 18 | /** 19 | * Current number of rows. 20 | * 21 | * For a partial result set, this may change as more rows are read. 22 | * For a full result, or a batch, this will not change. 23 | */ 24 | abstract get rowCount(): number; 25 | 26 | /** 27 | * Returns the name of column at the given index (starting at zero). 28 | * 29 | * Note that duplicate column names are possible. 30 | */ 31 | abstract columnName(columnIndex: number): string; 32 | 33 | /** 34 | * Returns the type of the column at the given index (starting at zero). 35 | */ 36 | abstract columnType(columnIndex: number): DuckDBType; 37 | 38 | /** 39 | * Returns the value for the given column and row. Both are zero-indexed. 40 | */ 41 | abstract value(columnIndex: number, rowIndex: number): DuckDBValue; 42 | 43 | /** 44 | * Returns the single value, assuming exactly one column and row. Throws otherwise. 45 | */ 46 | singleValue(): DuckDBValue { 47 | const { columnCount, rowCount } = this; 48 | if (columnCount === 0) { 49 | throw Error('no column data'); 50 | } 51 | if (rowCount === 0) { 52 | throw Error('no rows'); 53 | } 54 | if (columnCount > 1) { 55 | throw Error('more than one column'); 56 | } 57 | if (rowCount > 1) { 58 | throw Error('more than one row'); 59 | } 60 | return this.value(0, 0); 61 | } 62 | 63 | /** 64 | * Returns the column names as an array. 65 | */ 66 | columnNames(): readonly string[] { 67 | const { columnCount } = this; 68 | const outputColumnNames: string[] = []; 69 | for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { 70 | outputColumnNames.push(this.columnName(columnIndex)); 71 | } 72 | return outputColumnNames; 73 | } 74 | 75 | /** 76 | * Returns the column names as an array, deduplicated following DuckDB's "Auto-Increment Duplicate Column Names" 77 | * behavior. 78 | */ 79 | deduplicatedColumnNames(): readonly string[] { 80 | const { columnCount } = this; 81 | const outputColumnNames: string[] = []; 82 | const columnNameCount: { [columnName: string]: number } = {}; 83 | for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { 84 | const inputColumnName = this.columnName(columnIndex); 85 | const nameCount = (columnNameCount[inputColumnName] || 0) + 1; 86 | columnNameCount[inputColumnName] = nameCount; 87 | if (nameCount > 1) { 88 | outputColumnNames.push(`${inputColumnName}:${nameCount - 1}`); 89 | } else { 90 | outputColumnNames.push(inputColumnName); 91 | } 92 | } 93 | return outputColumnNames; 94 | } 95 | 96 | /** 97 | * Returns the data as an array of row objects, keyed by column names. 98 | * 99 | * The column names are deduplicated following DuckDB's "Auto-Increment Duplicate Column Names" behavior. 100 | */ 101 | toRows(): readonly DuckDBRow[] { 102 | const { rowCount, columnCount } = this; 103 | const outputColumnNames = this.deduplicatedColumnNames(); 104 | const outputRows: DuckDBRow[] = []; 105 | for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) { 106 | const row: { [columnName: string]: DuckDBValue } = {}; 107 | for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { 108 | row[outputColumnNames[columnIndex]] = this.value(columnIndex, rowIndex); 109 | } 110 | outputRows.push(row); 111 | } 112 | return outputRows; 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/include/utils/helpers.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #ifndef DUCKDB_CPP_EXTENSION_ENTRY 5 | #include 6 | #endif 7 | #include 8 | 9 | // TODO we cannot run these checks because they are not defined for DuckDB < 1.4.x 10 | // #ifndef DUCKDB_MAJOR_VERSION 11 | // #error "DUCKDB_MAJOR_VERSION is not defined" 12 | // ... 13 | #define DUCKDB_VERSION_AT_MOST(major, minor, patch) \ 14 | (DUCKDB_MAJOR_VERSION < (major) || (DUCKDB_MAJOR_VERSION == (major) && DUCKDB_MINOR_VERSION < (minor)) || \ 15 | (DUCKDB_MAJOR_VERSION == (major) && DUCKDB_MINOR_VERSION == (minor) && DUCKDB_PATCH_VERSION <= (patch))) 16 | 17 | namespace duckdb { 18 | 19 | typedef std::string (*simple_tf_t)(ClientContext &); 20 | 21 | struct RunOnceTableFunctionState : GlobalTableFunctionState { 22 | RunOnceTableFunctionState() : run(false) {}; 23 | std::atomic run; 24 | 25 | static unique_ptr Init(ClientContext &, 26 | TableFunctionInitInput &) { 27 | return make_uniq(); 28 | } 29 | }; 30 | 31 | namespace internal { 32 | 33 | unique_ptr SingleBoolResultBind(ClientContext &, 34 | TableFunctionBindInput &, 35 | vector &out_types, 36 | vector &out_names); 37 | 38 | unique_ptr SingleStringResultBind(ClientContext &, 39 | TableFunctionBindInput &, 40 | vector &, 41 | vector &); 42 | 43 | bool ShouldRun(TableFunctionInput &input); 44 | 45 | template struct CallFunctionHelper; 46 | 47 | template <> struct CallFunctionHelper { 48 | static std::string call(ClientContext &context, TableFunctionInput &input, 49 | std::string (*f)(ClientContext &)) { 50 | return f(context); 51 | } 52 | }; 53 | 54 | template <> 55 | struct CallFunctionHelper { 57 | static std::string call(ClientContext &context, TableFunctionInput &input, 58 | std::string (*f)(ClientContext &, 59 | TableFunctionInput &)) { 60 | return f(context, input); 61 | } 62 | }; 63 | 64 | template 65 | void TableFunc(ClientContext &context, TableFunctionInput &input, 66 | DataChunk &output) { 67 | if (!ShouldRun(input)) { 68 | return; 69 | } 70 | 71 | const std::string result = 72 | CallFunctionHelper::call(context, input, func); 73 | output.SetCardinality(1); 74 | output.SetValue(0, 0, result); 75 | } 76 | 77 | #ifdef DUCKDB_CPP_EXTENSION_ENTRY 78 | template 79 | void RegisterTF(ExtensionLoader &loader, const char *name) { 80 | TableFunction tf(name, {}, internal::TableFunc, 81 | internal::SingleStringResultBind, 82 | RunOnceTableFunctionState::Init); 83 | loader.RegisterFunction(tf); 84 | } 85 | #else 86 | template 87 | void RegisterTF(DatabaseInstance &instance, const char *name) { 88 | TableFunction tf(name, {}, internal::TableFunc, 89 | internal::SingleStringResultBind, 90 | RunOnceTableFunctionState::Init); 91 | ExtensionUtil::RegisterFunction(instance, tf); 92 | } 93 | #endif 94 | 95 | } // namespace internal 96 | 97 | #ifdef DUCKDB_CPP_EXTENSION_ENTRY 98 | #define REGISTER_TF(name, func) \ 99 | internal::RegisterTF(loader, name) 100 | #else 101 | #define REGISTER_TF(name, func) \ 102 | internal::RegisterTF(instance, name) 103 | #endif 104 | 105 | } // namespace duckdb 106 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/src/serialization/functions/typeReaders.ts: -------------------------------------------------------------------------------- 1 | import { BinaryDeserializer } from '../classes/BinaryDeserializer.js'; 2 | import { BaseTypeInfo, TypeIdAndInfo, TypeInfo } from '../types/TypeInfo.js'; 3 | import { 4 | readList, 5 | readNullable, 6 | readPair, 7 | readString, 8 | readStringList, 9 | readUint8, 10 | readUnsupported, 11 | readVarInt, 12 | } from './basicReaders.js'; 13 | 14 | export function readStructEntry( 15 | deserializer: BinaryDeserializer, 16 | ): [string, TypeIdAndInfo] { 17 | return readPair(deserializer, readString, readType); 18 | } 19 | 20 | export function readStructEntryList( 21 | deserializer: BinaryDeserializer, 22 | ): [string, TypeIdAndInfo][] { 23 | return readList(deserializer, readStructEntry); 24 | } 25 | 26 | /** See ExtraTypeInfo::Deserialize in https://github.com/duckdb/duckdb/blob/main/src/storage/serialization/serialize_types.cpp */ 27 | export function readTypeInfo(deserializer: BinaryDeserializer): TypeInfo { 28 | const typeInfoType = deserializer.readProperty(100, readUint8); 29 | const alias = deserializer.readPropertyWithDefault(101, readString, null); 30 | const modifiers = deserializer.readPropertyWithDefault( 31 | 102, 32 | readUnsupported, 33 | null, 34 | ); 35 | const baseInfo: BaseTypeInfo = { 36 | ...(alias ? { alias } : {}), 37 | ...(modifiers ? { modifiers } : {}), 38 | }; 39 | let typeInfo: TypeInfo | undefined; 40 | switch (typeInfoType) { 41 | case 1: // GENERIC_TYPE_INFO 42 | typeInfo = { 43 | ...baseInfo, 44 | kind: 'generic', 45 | }; 46 | break; 47 | case 2: // DECIMAL_TYPE_INFO 48 | { 49 | const width = deserializer.readPropertyWithDefault(200, readUint8, 0); 50 | const scale = deserializer.readPropertyWithDefault(201, readUint8, 0); 51 | typeInfo = { 52 | ...baseInfo, 53 | kind: 'decimal', 54 | width, 55 | scale, 56 | }; 57 | } 58 | break; 59 | case 4: // LIST_TYPE_INFO 60 | { 61 | const childType = deserializer.readProperty(200, readType); 62 | typeInfo = { 63 | ...baseInfo, 64 | kind: 'list', 65 | childType, 66 | }; 67 | } 68 | break; 69 | case 5: // STRUCT_TYPE_INFO 70 | { 71 | const childTypes = deserializer.readProperty(200, readStructEntryList); 72 | typeInfo = { 73 | ...baseInfo, 74 | kind: 'struct', 75 | childTypes, 76 | }; 77 | } 78 | break; 79 | case 6: // ENUM_TYPE_INFO 80 | { 81 | const valuesCount = deserializer.readProperty(200, readVarInt); 82 | const values = deserializer.readProperty(201, readStringList); 83 | typeInfo = { 84 | ...baseInfo, 85 | kind: 'enum', 86 | valuesCount, 87 | values, 88 | }; 89 | } 90 | break; 91 | case 9: // ARRAY_TYPE_INFO 92 | { 93 | const childType = deserializer.readProperty(200, readType); 94 | const size = deserializer.readPropertyWithDefault(201, readVarInt, 0); 95 | typeInfo = { 96 | ...baseInfo, 97 | kind: 'array', 98 | childType, 99 | size, 100 | }; 101 | } 102 | break; 103 | default: 104 | throw new Error(`unsupported type info: ${typeInfoType}`); 105 | } 106 | deserializer.expectObjectEnd(); 107 | if (!typeInfo) { 108 | typeInfo = { 109 | ...baseInfo, 110 | kind: 'generic', 111 | }; 112 | } 113 | return typeInfo; 114 | } 115 | 116 | export function readNullableTypeInfo( 117 | deserializer: BinaryDeserializer, 118 | ): TypeInfo | null { 119 | return readNullable(deserializer, readTypeInfo); 120 | } 121 | 122 | export function readType(deserializer: BinaryDeserializer): TypeIdAndInfo { 123 | const id = deserializer.readProperty(100, readUint8); 124 | const typeInfo = deserializer.readPropertyWithDefault( 125 | 101, 126 | readNullableTypeInfo, 127 | null, 128 | ); 129 | deserializer.expectObjectEnd(); 130 | return { id, ...(typeInfo ? { typeInfo } : {}) }; 131 | } 132 | 133 | export function readTypeList( 134 | deserializer: BinaryDeserializer, 135 | ): TypeIdAndInfo[] { 136 | return readList(deserializer, readType); 137 | } 138 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBBlobValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBBlobValue } from '../src/DuckDBBlobValue'; 3 | 4 | suite('DuckDBBlobValue', () => { 5 | test('should render an empty byte array to the correct string', () => { 6 | expect(new DuckDBBlobValue(new Uint8Array([])).toString()).toStrictEqual( 7 | '', 8 | ); 9 | }); 10 | test('should render a byte array to the correct string', () => { 11 | expect( 12 | new DuckDBBlobValue( 13 | new Uint8Array([0x41, 0x42, 0x43, 0x31, 0x32, 0x33]), 14 | ).toString(), 15 | ).toStrictEqual('ABC123'); 16 | }); 17 | test('should render a byte array containing single-digit non-printables to the correct string', () => { 18 | expect( 19 | new DuckDBBlobValue( 20 | new Uint8Array([ 21 | 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 22 | 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 23 | ]), 24 | ).toString(), 25 | ).toStrictEqual( 26 | '\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0A\\x0B\\x0C\\x0D\\x0E\\x0F', 27 | ); 28 | }); 29 | test('should render a byte array containing double-digit non-printables to the correct string', () => { 30 | expect( 31 | new DuckDBBlobValue( 32 | new Uint8Array([ 33 | 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 34 | 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 35 | ]), 36 | ).toString(), 37 | ).toStrictEqual( 38 | '\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1A\\x1B\\x1C\\x1D\\x1E\\x1F', 39 | ); 40 | }); 41 | test('should render a byte array containing min printables (including single and double quotes) to the correct string', () => { 42 | expect( 43 | new DuckDBBlobValue( 44 | new Uint8Array([ 45 | 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 46 | 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 47 | ]), 48 | ).toString(), 49 | ).toStrictEqual(' !\\x22#$%&\\x27()*+,-./'); 50 | }); 51 | test('should render a byte array containing max printables (including backspace) to the correct string', () => { 52 | expect( 53 | new DuckDBBlobValue( 54 | new Uint8Array([ 55 | 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 56 | 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 57 | ]), 58 | ).toString(), 59 | ).toStrictEqual('pqrstuvwxyz{|}~\\x7F'); 60 | }); 61 | test('should render a byte array containing high non-printables to the correct string', () => { 62 | expect( 63 | new DuckDBBlobValue( 64 | new Uint8Array([ 65 | 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 66 | 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 67 | ]), 68 | ).toString(), 69 | ).toStrictEqual( 70 | '\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8A\\x8B\\x8C\\x8D\\x8E\\x8F', 71 | ); 72 | }); 73 | test('should render a byte array containing max non-printables to the correct string', () => { 74 | expect( 75 | new DuckDBBlobValue( 76 | new Uint8Array([ 77 | 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 78 | 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 79 | ]), 80 | ).toString(), 81 | ).toStrictEqual( 82 | '\\xF0\\xF1\\xF2\\xF3\\xF4\\xF5\\xF6\\xF7\\xF8\\xF9\\xFA\\xFB\\xFC\\xFD\\xFE\\xFF', 83 | ); 84 | }); 85 | test('toJson', () => { 86 | expect( 87 | new DuckDBBlobValue( 88 | new Uint8Array([0x41, 0x42, 0x43, 0x31, 0x32, 0x33]), 89 | ).toJson(), 90 | ).toStrictEqual('ABC123'); 91 | }); 92 | 93 | suite('toSql', () => { 94 | test('should render empty blob to SQL', () => { 95 | expect(new DuckDBBlobValue(new Uint8Array([])).toSql()).toStrictEqual( 96 | "''::BLOB", 97 | ); 98 | }); 99 | 100 | test('should render blob value to SQL', () => { 101 | const bytes = new Uint8Array([0x48, 0x65, 0x6c, 0x6c, 0x6f]); // "Hello" 102 | expect(new DuckDBBlobValue(bytes).toSql()).toStrictEqual("'Hello'::BLOB"); 103 | }); 104 | 105 | test('should render blob with printable ASCII to SQL', () => { 106 | const bytes = new Uint8Array([0x41, 0x42, 0x43, 0x31, 0x32, 0x33]); // "ABC123" 107 | expect(new DuckDBBlobValue(bytes).toSql()).toStrictEqual( 108 | "'ABC123'::BLOB", 109 | ); 110 | }); 111 | 112 | test('should render blob with non-printable bytes to SQL', () => { 113 | const bytes = new Uint8Array([0x00, 0x01, 0x02, 0xff]); 114 | expect(new DuckDBBlobValue(bytes).toSql()).toStrictEqual( 115 | "'\\x00\\x01\\x02\\xFF'::BLOB", 116 | ); 117 | }); 118 | }); 119 | }); 120 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/test/DuckDBMapValue.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, suite, test } from 'vitest'; 2 | import { DuckDBListValue } from '../src/DuckDBListValue'; 3 | import { DuckDBMapValue } from '../src/DuckDBMapValue'; 4 | 5 | suite('DuckDBMapValue', () => { 6 | test('should render an empty map to the correct string', () => { 7 | expect(new DuckDBMapValue([]).toString()).toStrictEqual('{}'); 8 | }); 9 | test('should render a single-entry map to the correct string', () => { 10 | expect( 11 | new DuckDBMapValue([{ key: 'x', value: 1 }]).toString(), 12 | ).toStrictEqual(`{'x': 1}`); 13 | }); 14 | test('should render a multi-entry map to the correct string', () => { 15 | expect( 16 | new DuckDBMapValue([ 17 | { key: 1, value: 42.001 }, 18 | { key: 5, value: -32.1 }, 19 | { key: 3, value: null }, 20 | ]).toString(), 21 | ).toStrictEqual(`{1: 42.001, 5: -32.1, 3: NULL}`); 22 | }); 23 | test('should render a multi-entry map with complex key types to the correct string', () => { 24 | expect( 25 | new DuckDBMapValue([ 26 | { 27 | key: new DuckDBListValue(['a', 'b']), 28 | value: new DuckDBListValue([1.1, 2.2]), 29 | }, 30 | { 31 | key: new DuckDBListValue(['c', 'd']), 32 | value: new DuckDBListValue([3.3, 4.4]), 33 | }, 34 | ]).toString(), 35 | ).toStrictEqual(`{['a', 'b']: [1.1, 2.2], ['c', 'd']: [3.3, 4.4]}`); 36 | }); 37 | test('should render a map with nested maps to the correct string', () => { 38 | expect( 39 | new DuckDBMapValue([ 40 | { key: new DuckDBMapValue([]), value: new DuckDBMapValue([]) }, 41 | { 42 | key: new DuckDBMapValue([{ key: 'key1', value: 'value1' }]), 43 | value: new DuckDBMapValue([ 44 | { key: 1, value: 42.001 }, 45 | { key: 5, value: -32.1 }, 46 | { key: 3, value: null }, 47 | ]), 48 | }, 49 | ]).toString(), 50 | ).toStrictEqual( 51 | `{{}: {}, {'key1': 'value1'}: {1: 42.001, 5: -32.1, 3: NULL}}`, 52 | ); 53 | }); 54 | test('toJson basics', () => { 55 | expect( 56 | new DuckDBMapValue([ 57 | { key: 'a', value: 1 }, 58 | { key: 'b', value: 2 }, 59 | { key: 'c', value: 3 }, 60 | ]).toJson(), 61 | ).toStrictEqual({ "'a'": 1, "'b'": 2, "'c'": 3 }); 62 | }); 63 | test('toJson with complex keys and values', () => { 64 | expect( 65 | new DuckDBMapValue([ 66 | { 67 | key: new DuckDBListValue(['a', 'b']), 68 | value: new DuckDBListValue([1.1, 2.2]), 69 | }, 70 | { 71 | key: new DuckDBListValue(['c', 'd']), 72 | value: new DuckDBListValue([3.3, 4.4]), 73 | }, 74 | ]).toJson(), 75 | ).toStrictEqual({ "['a', 'b']": [1.1, 2.2], "['c', 'd']": [3.3, 4.4] }); 76 | }); 77 | 78 | suite('toSql', () => { 79 | test('should render empty map', () => { 80 | expect(new DuckDBMapValue([]).toSql()).toStrictEqual('MAP {}'); 81 | }); 82 | 83 | test('should render map with entries', () => { 84 | expect( 85 | new DuckDBMapValue([ 86 | { key: 'foo', value: 123 }, 87 | { key: 'bar', value: 'abc' }, 88 | ]).toSql(), 89 | ).toStrictEqual("MAP {'foo': 123, 'bar': 'abc'}"); 90 | }); 91 | 92 | test('should render map with null values', () => { 93 | expect( 94 | new DuckDBMapValue([ 95 | { key: 'a', value: null }, 96 | { key: 'b', value: 456 }, 97 | ]).toSql(), 98 | ).toStrictEqual("MAP {'a': NULL, 'b': 456}"); 99 | }); 100 | 101 | test('should render nested maps', () => { 102 | expect( 103 | new DuckDBMapValue([ 104 | { 105 | key: 'nested', 106 | value: new DuckDBMapValue([{ key: 'inner', value: 42 }]), 107 | }, 108 | ]).toSql(), 109 | ).toStrictEqual("MAP {'nested': MAP {'inner': 42}}"); 110 | }); 111 | 112 | test('should render map with numeric keys', () => { 113 | expect( 114 | new DuckDBMapValue([ 115 | { key: 1, value: 'one' }, 116 | { key: 2, value: 'two' }, 117 | { key: 3, value: 'three' }, 118 | ]).toSql(), 119 | ).toStrictEqual("MAP {1: 'one', 2: 'two', 3: 'three'}"); 120 | }); 121 | 122 | test('should render map with mixed key types', () => { 123 | expect( 124 | new DuckDBMapValue([ 125 | { key: 100, value: 'hundred' }, 126 | { key: 'key', value: 'value' }, 127 | { key: true, value: 'boolean key' }, 128 | ]).toSql(), 129 | ).toStrictEqual( 130 | "MAP {100: 'hundred', 'key': 'value', TRUE: 'boolean key'}", 131 | ); 132 | }); 133 | }); 134 | }); 135 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-data-values/src/conversion/stringFromDecimal.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Decimal string formatting. 3 | * 4 | * Supports a subset of the functionality of `BigInt.prototype.toLocaleString` for locale-specific formatting. 5 | */ 6 | 7 | /* 8 | * Locale formatting options for DuckDBDecimalValue. 9 | * 10 | * This is a subset of the options available for `BigInt.prototype.toLocaleString` 11 | */ 12 | export interface DuckDBDecimalFormatOptions { 13 | useGrouping?: boolean; 14 | minimumFractionDigits?: number; 15 | maximumFractionDigits?: number; 16 | } 17 | 18 | export interface LocaleOptions { 19 | locales?: string | string[]; 20 | options?: DuckDBDecimalFormatOptions; 21 | } 22 | 23 | /* 24 | * Get the decimal separator for a given locale. 25 | * Somewhat expensive, so use getCachedDecimalSeparator if you need to call this multiple times. 26 | */ 27 | 28 | function getDecimalSeparator(locales?: string | string[]): string { 29 | const decimalSeparator = 30 | new Intl.NumberFormat(locales, { useGrouping: false }) 31 | .formatToParts(0.1) 32 | .find((part) => part.type === 'decimal')?.value ?? '.'; 33 | return decimalSeparator; 34 | } 35 | 36 | /* 37 | * Get the decimal separator for a given locale, and cache the result. 38 | */ 39 | const cachedDecimalSeparators: { [localeKey: string]: string } = {}; 40 | 41 | function getCachedDecimalSeparator(locales?: string | string[]): string { 42 | const cacheKey = JSON.stringify(locales); 43 | if (cacheKey in cachedDecimalSeparators) { 44 | return cachedDecimalSeparators[cacheKey]; 45 | } 46 | const decimalSeparator = getDecimalSeparator(locales); 47 | cachedDecimalSeparators[cacheKey] = decimalSeparator; 48 | return decimalSeparator; 49 | } 50 | 51 | // Helper function to format whole part of a decimal value. 52 | // Note that we explicitly omit 'minimumFractionDigits' and 'maximumFractionDigits' from the options 53 | // passed to toLocaleString, because they are only relevant for the fractional part of the number, and 54 | // would result in formatting the whole part as a real number, which we don't want. 55 | function formatWholePart( 56 | localeOptions: LocaleOptions | undefined, 57 | val: bigint, 58 | ): string { 59 | if (localeOptions) { 60 | const { 61 | minimumFractionDigits: _minFD, 62 | maximumFractionDigits: _maxFD, 63 | ...restOptions 64 | } = localeOptions.options ?? {}; 65 | return val.toLocaleString(localeOptions?.locales, restOptions); 66 | } 67 | return String(val); 68 | } 69 | 70 | // Format the fractional part of a decimal value 71 | // Note that we must handle minimumFractionDigits and maximumFractionDigits ourselves, and that 72 | // we don't apply `useGrouping` because that only applies to the whole part of the number. 73 | function formatFractionalPart( 74 | localeOptions: LocaleOptions | undefined, 75 | val: bigint, 76 | scale: number, 77 | ): string { 78 | const fractionalPartStr = String(val).padStart(scale, '0'); 79 | if (!localeOptions) { 80 | return fractionalPartStr; 81 | } 82 | const minFracDigits = localeOptions?.options?.minimumFractionDigits ?? 0; 83 | const maxFracDigits = localeOptions?.options?.maximumFractionDigits ?? 20; 84 | 85 | return fractionalPartStr.padEnd(minFracDigits, '0').slice(0, maxFracDigits); 86 | } 87 | 88 | /** 89 | * Convert a scaled decimal value to a string, possibly using locale-specific formatting. 90 | */ 91 | export function stringFromDecimal( 92 | scaledValue: bigint, 93 | scale: number, 94 | localeOptions?: LocaleOptions, 95 | ): string { 96 | // Decimal values are represented as integers that have been scaled up by a power of ten. The `scale` property of 97 | // the type is the exponent of the scale factor. For a scale greater than zero, we need to separate out the 98 | // fractional part by reversing this scaling. 99 | if (scale > 0) { 100 | const scaleFactor = BigInt(10) ** BigInt(scale); 101 | const absScaledValue = scaledValue < 0 ? -scaledValue : scaledValue; 102 | 103 | const prefix = scaledValue < 0 ? '-' : ''; 104 | 105 | const wholePartNum = absScaledValue / scaleFactor; 106 | const wholePartStr = formatWholePart(localeOptions, wholePartNum); 107 | 108 | const fractionalPartNum = absScaledValue % scaleFactor; 109 | const fractionalPartStr = formatFractionalPart( 110 | localeOptions, 111 | fractionalPartNum, 112 | scale, 113 | ); 114 | 115 | const decimalSeparatorStr = localeOptions 116 | ? getCachedDecimalSeparator(localeOptions.locales) 117 | : '.'; 118 | 119 | return `${prefix}${wholePartStr}${decimalSeparatorStr}${fractionalPartStr}`; 120 | } 121 | // For a scale of zero, there is no fractional part, so a direct string conversion works. 122 | if (localeOptions) { 123 | return scaledValue.toLocaleString( 124 | localeOptions?.locales, 125 | localeOptions?.options as BigIntToLocaleStringOptions | undefined, 126 | ); 127 | } 128 | return String(scaledValue); 129 | } 130 | -------------------------------------------------------------------------------- /ts/pkgs/duckdb-ui-client/test/http/classes/DuckDBUIHttpRequestQueue.test.ts: -------------------------------------------------------------------------------- 1 | import { http, HttpResponse } from 'msw'; 2 | import { expect, suite, test } from 'vitest'; 3 | import { DuckDBUIHttpRequestQueue } from '../../../src/http/classes/DuckDBUIHttpRequestQueue'; 4 | import { makeBuffer } from '../../helpers/makeBuffer'; 5 | import { mockRequests } from '../../helpers/mockRequests'; 6 | 7 | suite('DuckDBUIHttpRequestQueue', () => { 8 | test('single request', () => { 9 | return mockRequests( 10 | [ 11 | http.post('http://localhost/example/path', () => { 12 | return HttpResponse.arrayBuffer(makeBuffer([17, 42])); 13 | }), 14 | ], 15 | async () => { 16 | const queue = new DuckDBUIHttpRequestQueue(); 17 | const id = queue.enqueue( 18 | 'http://localhost/example/path', 19 | 'example body', 20 | ); 21 | expect(queue.length).toBe(1); 22 | expect(queue.isCurrent(id)).toBe(true); 23 | 24 | const result = await queue.enqueuedResult(id); 25 | expect(result.buffer).toEqual(makeBuffer([17, 42])); 26 | }, 27 | ); 28 | }); 29 | test('multiple requests', () => { 30 | return mockRequests( 31 | [ 32 | http.post('http://localhost/example/path', async ({ request }) => { 33 | const body = await request.text(); 34 | const value = parseInt(body.split(' ')[0], 10); 35 | return HttpResponse.arrayBuffer(makeBuffer([value])); 36 | }), 37 | ], 38 | async () => { 39 | const queue = new DuckDBUIHttpRequestQueue(); 40 | const id1 = queue.enqueue( 41 | 'http://localhost/example/path', 42 | '11 example body', 43 | ); 44 | const id2 = queue.enqueue( 45 | 'http://localhost/example/path', 46 | '22 example body', 47 | ); 48 | expect(queue.length).toBe(2); 49 | expect(queue.isCurrent(id1)).toBe(true); 50 | 51 | const result1 = await queue.enqueuedResult(id1); 52 | expect(result1.buffer).toEqual(makeBuffer([11])); 53 | 54 | expect(queue.length).toBe(1); 55 | expect(queue.isCurrent(id2)).toBe(true); 56 | 57 | const result2 = await queue.enqueuedResult(id2); 58 | expect(result2.buffer).toEqual(makeBuffer([22])); 59 | }, 60 | ); 61 | }); 62 | test('cancel (first request)', () => { 63 | return mockRequests( 64 | [ 65 | http.post('http://localhost/example/path', async ({ request }) => { 66 | const body = await request.text(); 67 | const value = parseInt(body.split(' ')[0], 10); 68 | return HttpResponse.arrayBuffer(makeBuffer([value])); 69 | }), 70 | ], 71 | async () => { 72 | const queue = new DuckDBUIHttpRequestQueue(); 73 | const id1 = queue.enqueue( 74 | 'http://localhost/example/path', 75 | '11 example body', 76 | ); 77 | const id2 = queue.enqueue( 78 | 'http://localhost/example/path', 79 | '22 example body', 80 | ); 81 | expect(queue.length).toBe(2); 82 | expect(queue.isCurrent(id1)).toBe(true); 83 | 84 | queue.cancel(id1); 85 | await expect(queue.enqueuedResult(id1)).rejects.toEqual( 86 | new Error('query was canceled'), 87 | ); 88 | 89 | const result2 = await queue.enqueuedResult(id2); 90 | expect(result2.buffer).toEqual(makeBuffer([22])); 91 | }, 92 | ); 93 | }); 94 | test('cancel (second request)', () => { 95 | return mockRequests( 96 | [ 97 | http.post('http://localhost/example/path', async ({ request }) => { 98 | const body = await request.text(); 99 | const value = parseInt(body.split(' ')[0], 10); 100 | return HttpResponse.arrayBuffer(makeBuffer([value])); 101 | }), 102 | ], 103 | async () => { 104 | const queue = new DuckDBUIHttpRequestQueue(); 105 | const id1 = queue.enqueue( 106 | 'http://localhost/example/path', 107 | '11 example body', 108 | ); 109 | const id2 = queue.enqueue( 110 | 'http://localhost/example/path', 111 | '22 example body', 112 | ); 113 | const id3 = queue.enqueue( 114 | 'http://localhost/example/path', 115 | '33 example body', 116 | ); 117 | expect(queue.length).toBe(3); 118 | expect(queue.isCurrent(id1)).toBe(true); 119 | 120 | const promise2 = queue.enqueuedResult(id2); 121 | queue.cancel(id2, 'example error message'); 122 | 123 | const result1 = await queue.enqueuedResult(id1); 124 | expect(result1.buffer).toEqual(makeBuffer([11])); 125 | 126 | expect(queue.length).toBe(1); 127 | expect(queue.isCurrent(id3)).toBe(true); 128 | 129 | await expect(promise2).rejects.toEqual( 130 | new Error('example error message'), 131 | ); 132 | 133 | const result3 = await queue.enqueuedResult(id3); 134 | expect(result3.buffer).toEqual(makeBuffer([33])); 135 | }, 136 | ); 137 | }); 138 | }); 139 | --------------------------------------------------------------------------------