├── .release-please-manifest.json ├── CODEOWNERS ├── .prettierrc ├── src ├── http │ ├── HttpClient.ts │ ├── types.ts │ └── createFetchClient.ts ├── utils │ └── url.ts ├── index.ts ├── errors │ └── IcebergError.ts └── catalog │ ├── namespaces.ts │ ├── tables.ts │ ├── types.ts │ └── IcebergRestCatalog.ts ├── .editorconfig ├── vitest.config.ts ├── test ├── compatibility │ ├── cjs-project │ │ ├── package.json │ │ └── index.js │ ├── esm-project │ │ ├── package.json │ │ └── index.js │ ├── ts-cjs-project │ │ ├── tsconfig.json │ │ ├── package.json │ │ └── index.ts │ ├── ts-esm-project │ │ ├── tsconfig.json │ │ ├── package.json │ │ └── index.ts │ ├── README.md │ └── run-all.sh ├── integration │ ├── TESTING-DOCKER.md │ └── test-local-catalog.integration.test.ts ├── catalog │ ├── namespaces.test.ts │ └── tables.test.ts └── http │ └── createFetchClient.test.ts ├── vitest.config.integration.ts ├── .gitignore ├── release-please-config.json ├── tsup.config.ts ├── .github ├── workflows │ ├── ci.yml │ ├── compatibility.yml │ ├── test.yml │ ├── docs.yml │ ├── preview-release.yml │ ├── integration-test.yml │ └── release.yml └── ISSUE_TEMPLATE │ ├── ✍️-feature-request.md │ └── 🐞-bug-report.md ├── tsconfig.json ├── eslint.config.ts ├── LICENSE ├── docker-compose.yml ├── package.json ├── examples ├── basic-usage.ts └── supabase-storage-usage.ts ├── scripts └── test-integration.sh ├── CHANGELOG.md ├── CONTRIBUTING.md └── README.md /.release-please-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | ".": "0.8.1" 3 | } 4 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Global owners for the entire repository 2 | * @supabase/sdk @supabase/admin @supabase/security -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "tabWidth": 2, 5 | "trailingComma": "es5", 6 | "printWidth": 100 7 | } 8 | -------------------------------------------------------------------------------- /src/http/HttpClient.ts: -------------------------------------------------------------------------------- 1 | export { createFetchClient } from './createFetchClient' 2 | export type { HttpClient, HttpRequest, HttpResponse, AuthConfig } from './types' 3 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitest/config' 2 | 3 | export default defineConfig({ 4 | test: { 5 | globals: true, 6 | environment: 'node', 7 | exclude: ['**/node_modules/**', '**/dist/**', '**/*.integration.test.ts'], 8 | }, 9 | }) 10 | -------------------------------------------------------------------------------- /test/compatibility/cjs-project/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-cjs-compatibility", 3 | "version": "1.0.0", 4 | "private": true, 5 | "scripts": { 6 | "test": "node index.js" 7 | }, 8 | "dependencies": { 9 | "iceberg-js": "file:../../.." 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /vitest.config.integration.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitest/config' 2 | 3 | export default defineConfig({ 4 | test: { 5 | globals: true, 6 | environment: 'node', 7 | include: ['test/integration/**/*.integration.test.ts'], 8 | testTimeout: 30000, 9 | }, 10 | }) 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | *.log 4 | .DS_Store 5 | coverage/ 6 | .env 7 | .env.local 8 | tmp/ 9 | 10 | docs/ 11 | 12 | # Compatibility test artifacts 13 | test/compatibility/**/package-lock.json 14 | test/compatibility/**/node_modules/ 15 | 16 | CLAUDE.md 17 | WARP.md 18 | .claude/ -------------------------------------------------------------------------------- /release-please-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": { 3 | ".": { 4 | "release-type": "node", 5 | "package-name": "iceberg-js", 6 | "changelog-path": "CHANGELOG.md", 7 | "bump-minor-pre-major": true, 8 | "bump-patch-for-minor-pre-major": false 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/compatibility/esm-project/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-esm-compatibility", 3 | "version": "1.0.0", 4 | "type": "module", 5 | "private": true, 6 | "scripts": { 7 | "test": "node index.js" 8 | }, 9 | "dependencies": { 10 | "iceberg-js": "file:../../.." 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /test/compatibility/ts-cjs-project/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "CommonJS", 5 | "moduleResolution": "node", 6 | "strict": true, 7 | "esModuleInterop": true, 8 | "skipLibCheck": true, 9 | "resolveJsonModule": true 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/compatibility/ts-esm-project/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "ESNext", 5 | "moduleResolution": "bundler", 6 | "strict": true, 7 | "esModuleInterop": true, 8 | "skipLibCheck": true, 9 | "resolveJsonModule": true 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/compatibility/ts-cjs-project/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-ts-cjs-compatibility", 3 | "version": "1.0.0", 4 | "private": true, 5 | "scripts": { 6 | "test": "tsx index.ts" 7 | }, 8 | "devDependencies": { 9 | "tsx": "^4.19.0", 10 | "typescript": "^5.9.3" 11 | }, 12 | "dependencies": { 13 | "iceberg-js": "file:../../.." 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /test/compatibility/ts-esm-project/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-ts-esm-compatibility", 3 | "version": "1.0.0", 4 | "type": "module", 5 | "private": true, 6 | "scripts": { 7 | "test": "tsx index.ts" 8 | }, 9 | "devDependencies": { 10 | "tsx": "^4.19.0", 11 | "typescript": "^5.9.3" 12 | }, 13 | "dependencies": { 14 | "iceberg-js": "file:../../.." 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'tsup' 2 | 3 | export default defineConfig({ 4 | entry: ['src/index.ts'], 5 | format: ['esm', 'cjs'], 6 | dts: true, 7 | sourcemap: true, 8 | clean: true, 9 | target: 'es2020', 10 | minify: false, 11 | treeshake: true, 12 | outExtension({ format }) { 13 | return { 14 | js: format === 'esm' ? '.mjs' : '.cjs', 15 | } 16 | }, 17 | }) 18 | -------------------------------------------------------------------------------- /src/utils/url.ts: -------------------------------------------------------------------------------- 1 | export function buildUrl( 2 | baseUrl: string, 3 | path: string, 4 | query?: Record 5 | ): string { 6 | const url = new URL(path, baseUrl) 7 | 8 | if (query) { 9 | for (const [key, value] of Object.entries(query)) { 10 | if (value !== undefined) { 11 | url.searchParams.set(key, value) 12 | } 13 | } 14 | } 15 | 16 | return url.toString() 17 | } 18 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | types: [opened, synchronize, reopened] 9 | 10 | jobs: 11 | test: 12 | name: Run Tests 13 | uses: ./.github/workflows/test.yml 14 | 15 | integration-test: 16 | name: Run Integration Tests 17 | uses: ./.github/workflows/integration-test.yml 18 | 19 | compatibility-test: 20 | name: Run Compatibility Tests 21 | uses: ./.github/workflows/compatibility.yml 22 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "ESNext", 5 | "moduleResolution": "bundler", 6 | "strict": true, 7 | "noEmit": true, 8 | "esModuleInterop": true, 9 | "forceConsistentCasingInFileNames": true, 10 | "skipLibCheck": true, 11 | "resolveJsonModule": true, 12 | "allowSyntheticDefaultImports": true, 13 | "types": ["vitest/globals", "node"] 14 | }, 15 | "include": ["src", "test"], 16 | "exclude": ["test/compatibility"] 17 | } 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/✍️-feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "✍️ Feature request" 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when ... 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /test/compatibility/esm-project/index.js: -------------------------------------------------------------------------------- 1 | // ESM Test - tests that the package works in ESM mode 2 | import { IcebergRestCatalog } from 'iceberg-js' 3 | 4 | console.log('✅ ESM: Successfully imported IcebergRestCatalog') 5 | console.log('✅ ESM: Type:', typeof IcebergRestCatalog) 6 | console.log('✅ ESM: Constructor name:', IcebergRestCatalog.name) 7 | 8 | // Test instantiation 9 | try { 10 | const catalog = new IcebergRestCatalog({ 11 | baseUrl: 'http://localhost:8181', 12 | auth: { type: 'none' }, 13 | }) 14 | console.log('✅ ESM: Successfully created catalog instance') 15 | console.log('✅ ESM: ALL TESTS PASSED!') 16 | } catch (error) { 17 | console.error('❌ ESM: Failed to create instance:', error) 18 | process.exit(1) 19 | } 20 | -------------------------------------------------------------------------------- /test/compatibility/cjs-project/index.js: -------------------------------------------------------------------------------- 1 | // CommonJS Test - tests that the package works in CJS mode 2 | const { IcebergRestCatalog } = require('iceberg-js') 3 | 4 | console.log('✅ CJS: Successfully required IcebergRestCatalog') 5 | console.log('✅ CJS: Type:', typeof IcebergRestCatalog) 6 | console.log('✅ CJS: Constructor name:', IcebergRestCatalog.name) 7 | 8 | // Test instantiation 9 | try { 10 | const catalog = new IcebergRestCatalog({ 11 | baseUrl: 'http://localhost:8181', 12 | auth: { type: 'none' }, 13 | }) 14 | console.log('✅ CJS: Successfully created catalog instance') 15 | console.log('✅ CJS: ALL TESTS PASSED!') 16 | } catch (error) { 17 | console.error('❌ CJS: Failed to create instance:', error) 18 | process.exit(1) 19 | } 20 | -------------------------------------------------------------------------------- /src/http/types.ts: -------------------------------------------------------------------------------- 1 | export interface HttpRequest { 2 | method: 'GET' | 'POST' | 'DELETE' | 'PUT' | 'HEAD' 3 | path: string 4 | query?: Record 5 | body?: unknown 6 | headers?: Record 7 | } 8 | 9 | export interface HttpResponse { 10 | status: number 11 | headers: Headers 12 | data: T 13 | } 14 | 15 | export interface HttpClient { 16 | request(req: HttpRequest): Promise> 17 | } 18 | 19 | export type AuthConfig = 20 | | { type: 'none' } 21 | | { type: 'bearer'; token: string } 22 | | { type: 'header'; name: string; value: string } 23 | | { 24 | type: 'custom' 25 | getHeaders: () => Record | Promise> 26 | } 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/🐞-bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41E Bug report" 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **System Info** 27 | Output of `npx envinfo --system --npmPackages 'iceberg-js' --binaries --browsers` 28 | 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /test/compatibility/ts-cjs-project/index.ts: -------------------------------------------------------------------------------- 1 | // TypeScript CJS Test - tests that the package works in TS CJS mode with types 2 | import { IcebergRestCatalog, type IcebergRestCatalogOptions } from 'iceberg-js' 3 | 4 | console.log('✅ TS-CJS: Successfully imported IcebergRestCatalog and types') 5 | 6 | // Test type checking 7 | const config: IcebergRestCatalogOptions = { 8 | baseUrl: 'http://localhost:8181', 9 | auth: { type: 'none' }, 10 | } 11 | 12 | // Test instantiation 13 | try { 14 | const catalog = new IcebergRestCatalog(config) 15 | console.log('✅ TS-CJS: Successfully created catalog instance') 16 | console.log('✅ TS-CJS: Type inference working:', typeof catalog.listNamespaces === 'function') 17 | console.log('✅ TS-CJS: ALL TESTS PASSED!') 18 | } catch (error) { 19 | console.error('❌ TS-CJS: Failed:', error) 20 | process.exit(1) 21 | } 22 | -------------------------------------------------------------------------------- /test/compatibility/ts-esm-project/index.ts: -------------------------------------------------------------------------------- 1 | // TypeScript ESM Test - tests that the package works in TS ESM mode with types 2 | import { IcebergRestCatalog, type IcebergRestCatalogOptions } from 'iceberg-js' 3 | 4 | console.log('✅ TS-ESM: Successfully imported IcebergRestCatalog and types') 5 | 6 | // Test type checking 7 | const config: IcebergRestCatalogOptions = { 8 | baseUrl: 'http://localhost:8181', 9 | auth: { type: 'none' }, 10 | } 11 | 12 | // Test instantiation 13 | try { 14 | const catalog = new IcebergRestCatalog(config) 15 | console.log('✅ TS-ESM: Successfully created catalog instance') 16 | console.log('✅ TS-ESM: Type inference working:', typeof catalog.listNamespaces === 'function') 17 | console.log('✅ TS-ESM: ALL TESTS PASSED!') 18 | } catch (error) { 19 | console.error('❌ TS-ESM: Failed:', error) 20 | process.exit(1) 21 | } 22 | -------------------------------------------------------------------------------- /.github/workflows/compatibility.yml: -------------------------------------------------------------------------------- 1 | name: Compatibility Tests 2 | 3 | on: 4 | workflow_call: 5 | workflow_dispatch: 6 | 7 | jobs: 8 | compatibility-test: 9 | name: Compatibility Tests 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | node-version: ['20', '22'] 14 | 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v4 18 | 19 | - name: Install pnpm 20 | uses: pnpm/action-setup@v4 21 | with: 22 | version: 9 23 | 24 | - name: Setup Node.js ${{ matrix.node-version }} 25 | uses: actions/setup-node@v4 26 | with: 27 | node-version: ${{ matrix.node-version }} 28 | cache: 'pnpm' 29 | 30 | - name: Install dependencies 31 | run: pnpm install 32 | 33 | - name: Build 34 | run: pnpm run build 35 | 36 | - name: Run compatibility tests 37 | run: pnpm run test:compatibility 38 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | workflow_call: 5 | workflow_dispatch: 6 | 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | 11 | strategy: 12 | matrix: 13 | node-version: [20.x, 22.x] 14 | 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v4 18 | 19 | - name: Install pnpm 20 | uses: pnpm/action-setup@v4 21 | with: 22 | version: latest 23 | 24 | - name: Setup Node.js 25 | uses: actions/setup-node@v4 26 | with: 27 | node-version: ${{ matrix.node-version }} 28 | cache: 'pnpm' 29 | 30 | - name: Install dependencies 31 | run: pnpm i 32 | 33 | - name: Run linter 34 | run: pnpm run lint 35 | 36 | - name: Run type check 37 | run: pnpm run type-check 38 | 39 | - name: Run tests 40 | run: pnpm test 41 | 42 | - name: Build 43 | run: pnpm run build 44 | -------------------------------------------------------------------------------- /eslint.config.ts: -------------------------------------------------------------------------------- 1 | import js from '@eslint/js' 2 | import globals from 'globals' 3 | import tseslint from 'typescript-eslint' 4 | import json from '@eslint/json' 5 | import markdown from '@eslint/markdown' 6 | import { defineConfig, globalIgnores } from 'eslint/config' 7 | 8 | export default defineConfig([ 9 | globalIgnores(['dist/**/*'], 'Ignore Build Directory'), 10 | globalIgnores(['CLAUDE.md', 'WARP.md', 'tmp/**/*', 'test/compatibility/**/*', 'docs/**/*']), 11 | { 12 | files: ['**/*.{js,mjs,cjs,ts,mts,cts}'], 13 | plugins: { js }, 14 | extends: ['js/recommended'], 15 | languageOptions: { globals: { ...globals.browser, ...globals.node } }, 16 | }, 17 | tseslint.configs.recommended, 18 | { files: ['**/*.json'], plugins: { json }, language: 'json/json', extends: ['json/recommended'] }, 19 | { 20 | files: ['**/*.md'], 21 | plugins: { markdown }, 22 | language: 'markdown/gfm', 23 | extends: ['markdown/recommended'], 24 | }, 25 | ]) 26 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Deploy docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: write 11 | pages: write 12 | 13 | jobs: 14 | deploy: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout repository 19 | uses: actions/checkout@v4 20 | 21 | - name: Setup pnpm 22 | uses: pnpm/action-setup@v4 23 | with: 24 | version: 9 25 | 26 | - name: Setup Node.js 27 | uses: actions/setup-node@v4 28 | with: 29 | node-version: '22' 30 | cache: 'pnpm' 31 | 32 | - name: Install dependencies 33 | run: pnpm install --frozen-lockfile 34 | 35 | - name: Build docs 36 | run: pnpm run docs 37 | 38 | - name: Deploy to gh-pages 39 | uses: peaceiris/actions-gh-pages@v3 40 | with: 41 | github_token: ${{ secrets.GITHUB_TOKEN }} 42 | publish_dir: ./docs 43 | -------------------------------------------------------------------------------- /test/compatibility/README.md: -------------------------------------------------------------------------------- 1 | # Compatibility Tests 2 | 3 | This directory contains test projects to verify that `iceberg-js` works in all common Node.js environments. 4 | 5 | ## Test Scenarios 6 | 7 | 1. **esm-project/** - Pure JavaScript ESM (`"type": "module"`) 8 | 2. **cjs-project/** - Pure JavaScript CommonJS (no type field) 9 | 3. **ts-esm-project/** - TypeScript with ESM (`module: "ESNext"`) 10 | 4. **ts-cjs-project/** - TypeScript with CommonJS (`module: "CommonJS"`) 11 | 12 | ## Running Tests 13 | 14 | From the root of the `iceberg-js` project: 15 | 16 | ```bash 17 | # Build the package first 18 | pnpm build 19 | 20 | # Run all compatibility tests 21 | bash test/compatibility/run-all.sh 22 | ``` 23 | 24 | Or test individually: 25 | 26 | ```bash 27 | cd test/compatibility/esm-project 28 | npm install 29 | npm test 30 | ``` 31 | 32 | ## What Gets Tested 33 | 34 | - ✅ Package can be imported/required 35 | - ✅ Classes and types are available 36 | - ✅ Instances can be created 37 | - ✅ TypeScript types resolve correctly 38 | - ✅ No module resolution errors 39 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { IcebergRestCatalog } from './catalog/IcebergRestCatalog' 2 | export type { IcebergRestCatalogOptions, AccessDelegation } from './catalog/IcebergRestCatalog' 3 | 4 | export type { 5 | NamespaceIdentifier, 6 | NamespaceMetadata, 7 | TableIdentifier, 8 | TableSchema, 9 | TableField, 10 | StructField, 11 | IcebergType, 12 | PrimitiveType, 13 | StructType, 14 | ListType, 15 | MapType, 16 | PrimitiveTypeValue, 17 | PartitionSpec, 18 | PartitionField, 19 | SortOrder, 20 | SortField, 21 | CreateTableRequest, 22 | CreateNamespaceResponse, 23 | CommitTableResponse, 24 | UpdateTableRequest, 25 | DropTableRequest, 26 | TableMetadata, 27 | } from './catalog/types' 28 | 29 | export { 30 | getCurrentSchema, 31 | parseDecimalType, 32 | parseFixedType, 33 | isDecimalType, 34 | isFixedType, 35 | typesEqual, 36 | } from './catalog/types' 37 | 38 | export type { AuthConfig } from './http/types' 39 | 40 | export { IcebergError } from './errors/IcebergError' 41 | export type { IcebergErrorResponse } from './errors/IcebergError' 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Supabase 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/preview-release.yml: -------------------------------------------------------------------------------- 1 | name: Preview release 2 | 3 | permissions: 4 | pull-requests: write 5 | 6 | on: 7 | pull_request: 8 | types: [opened, synchronize, labeled] 9 | 10 | jobs: 11 | preview: 12 | if: > 13 | github.repository == 'supabase/iceberg-js' && 14 | (github.event_name == 'push' || 15 | (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'trigger: preview'))) 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Checkout code 19 | uses: actions/checkout@v4 20 | with: 21 | filter: tree:0 22 | fetch-depth: 0 23 | 24 | - name: Install pnpm 25 | uses: pnpm/action-setup@v4 26 | with: 27 | version: latest 28 | 29 | - name: Setup Node.js 30 | uses: actions/setup-node@v4 31 | with: 32 | node-version: 20 33 | cache: 'pnpm' 34 | 35 | - name: Install dependencies 36 | run: pnpm i 37 | 38 | - name: Build 39 | run: pnpm run build 40 | 41 | - name: Publish preview packages 42 | run: | 43 | npx pkg-pr-new@latest publish 44 | -------------------------------------------------------------------------------- /.github/workflows/integration-test.yml: -------------------------------------------------------------------------------- 1 | name: Integration Tests 2 | 3 | on: 4 | workflow_call: 5 | workflow_dispatch: 6 | 7 | jobs: 8 | integration-test: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - name: Checkout code 13 | uses: actions/checkout@v4 14 | 15 | - name: Setup Node.js 16 | uses: actions/setup-node@v4 17 | with: 18 | node-version: '20' 19 | 20 | - name: Install pnpm 21 | uses: pnpm/action-setup@v4 22 | with: 23 | version: 8 24 | 25 | - name: Get pnpm store directory 26 | shell: bash 27 | run: | 28 | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV 29 | 30 | - name: Setup pnpm cache 31 | uses: actions/cache@v4 32 | with: 33 | path: ${{ env.STORE_PATH }} 34 | key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} 35 | restore-keys: | 36 | ${{ runner.os }}-pnpm-store- 37 | 38 | - name: Install dependencies 39 | run: pnpm install 40 | 41 | - name: Build library 42 | run: pnpm build 43 | 44 | - name: Run integration tests 45 | run: pnpm test:integration:ci 46 | 47 | - name: Show Docker logs on failure 48 | if: failure() 49 | run: docker compose logs 50 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | minio: 3 | image: minio/minio 4 | container_name: minio 5 | ports: 6 | - '9000:9000' 7 | - '9001:9001' 8 | networks: 9 | default: 10 | aliases: 11 | - warehouse--table-s3.minio 12 | healthcheck: 13 | test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1 14 | interval: 5s 15 | timeout: 20s 16 | retries: 10 17 | environment: 18 | MINIO_ROOT_USER: supa-storage 19 | MINIO_ROOT_PASSWORD: secret1234 20 | MINIO_DOMAIN: minio 21 | command: server --console-address ":9001" /data 22 | volumes: 23 | - minio-data:/data 24 | 25 | minio-setup: 26 | image: minio/mc 27 | container_name: minio-setup 28 | depends_on: 29 | minio: 30 | condition: service_healthy 31 | entrypoint: > 32 | /bin/sh -c " 33 | /usr/bin/mc alias set supa-minio http://minio:9000 supa-storage secret1234; 34 | /usr/bin/mc mb supa-minio/warehouse--table-s3; 35 | /usr/bin/mc policy set public supa-minio/warehouse--table-s3; 36 | exit 0; 37 | " 38 | 39 | iceberg-rest: 40 | image: tabulario/iceberg-rest 41 | container_name: iceberg-rest 42 | depends_on: 43 | - minio-setup 44 | ports: 45 | - 8181:8181 46 | environment: 47 | - AWS_ACCESS_KEY_ID=supa-storage 48 | - AWS_SECRET_ACCESS_KEY=secret1234 49 | - AWS_REGION=us-east-1 50 | - CATALOG_WAREHOUSE=s3://warehouse--table-s3/ 51 | - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO 52 | - CATALOG_S3_ENDPOINT=http://minio:9000 53 | 54 | volumes: 55 | minio-data: 56 | -------------------------------------------------------------------------------- /src/errors/IcebergError.ts: -------------------------------------------------------------------------------- 1 | export interface IcebergErrorResponse { 2 | error: { 3 | message: string 4 | type: string 5 | code: number 6 | stack?: string[] 7 | } 8 | } 9 | 10 | export class IcebergError extends Error { 11 | readonly status: number 12 | readonly icebergType?: string 13 | readonly icebergCode?: number 14 | readonly details?: unknown 15 | readonly isCommitStateUnknown: boolean 16 | 17 | constructor( 18 | message: string, 19 | opts: { 20 | status: number 21 | icebergType?: string 22 | icebergCode?: number 23 | details?: unknown 24 | } 25 | ) { 26 | super(message) 27 | this.name = 'IcebergError' 28 | this.status = opts.status 29 | this.icebergType = opts.icebergType 30 | this.icebergCode = opts.icebergCode 31 | this.details = opts.details 32 | 33 | // Detect CommitStateUnknownException (500, 502, 504 during table commits) 34 | this.isCommitStateUnknown = 35 | opts.icebergType === 'CommitStateUnknownException' || 36 | ([500, 502, 504].includes(opts.status) && opts.icebergType?.includes('CommitState') === true) 37 | } 38 | 39 | /** 40 | * Returns true if the error is a 404 Not Found error. 41 | */ 42 | isNotFound(): boolean { 43 | return this.status === 404 44 | } 45 | 46 | /** 47 | * Returns true if the error is a 409 Conflict error. 48 | */ 49 | isConflict(): boolean { 50 | return this.status === 409 51 | } 52 | 53 | /** 54 | * Returns true if the error is a 419 Authentication Timeout error. 55 | */ 56 | isAuthenticationTimeout(): boolean { 57 | return this.status === 419 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | pull-requests: write 11 | id-token: write # required for npm trusted publishing 12 | 13 | jobs: 14 | release: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: googleapis/release-please-action@v4 19 | id: release 20 | 21 | # Only continue if a release was created 22 | - uses: actions/checkout@v4 23 | if: ${{ steps.release.outputs.release_created }} 24 | 25 | - uses: pnpm/action-setup@v4 26 | if: ${{ steps.release.outputs.release_created }} 27 | with: 28 | version: 9 29 | 30 | - uses: actions/setup-node@v5 31 | if: ${{ steps.release.outputs.release_created }} 32 | with: 33 | node-version: 22 34 | registry-url: 'https://registry.npmjs.org' 35 | 36 | # Install latest npm globally in a user-writable prefix (no sudo, no perms issues) 37 | - name: Install latest npm (user prefix) 38 | if: ${{ steps.release.outputs.release_created }} 39 | run: | 40 | mkdir -p "$HOME/.npm-global" 41 | npm config set prefix "$HOME/.npm-global" 42 | echo "$HOME/.npm-global/bin" >> "$GITHUB_PATH" 43 | 44 | npm install -g npm@latest 45 | npm --version 46 | 47 | - name: Install dependencies 48 | if: ${{ steps.release.outputs.release_created }} 49 | run: pnpm install --frozen-lockfile 50 | 51 | - name: Build 52 | if: ${{ steps.release.outputs.release_created }} 53 | run: pnpm build 54 | 55 | # Use npm for publish (trusted publishing via OIDC) 56 | - name: Publish to npm (trusted publishing) 57 | if: ${{ steps.release.outputs.release_created }} 58 | run: npm publish --access public --provenance 59 | -------------------------------------------------------------------------------- /test/compatibility/run-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # Get the directory where the script is located 5 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" 7 | 8 | echo "SCRIPT_DIR: $SCRIPT_DIR" 9 | echo "ROOT_DIR: $ROOT_DIR" 10 | 11 | echo "🧪 Running compatibility tests for iceberg-js..." 12 | echo "" 13 | 14 | # Make sure the package is built 15 | if [ ! -d "$ROOT_DIR/dist" ]; then 16 | echo "❌ Error: dist/ folder not found. Run 'pnpm build' first." 17 | exit 1 18 | fi 19 | 20 | # Change to test/compatibility directory 21 | cd "$SCRIPT_DIR" 22 | 23 | # Test ESM 24 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 25 | echo "🔵 Testing ESM (Pure JavaScript)" 26 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 27 | cd esm-project 28 | npm install "$ROOT_DIR" 29 | npm test 30 | cd .. 31 | echo "" 32 | 33 | # Test CJS 34 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 35 | echo "🟢 Testing CommonJS (Pure JavaScript)" 36 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 37 | cd cjs-project 38 | npm install "$ROOT_DIR" 39 | npm test 40 | cd .. 41 | echo "" 42 | 43 | # Test TS ESM 44 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 45 | echo "🔷 Testing TypeScript ESM" 46 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 47 | cd ts-esm-project 48 | npm install 49 | npm install "$ROOT_DIR" 50 | npm test 51 | cd .. 52 | echo "" 53 | 54 | # Test TS CJS 55 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 56 | echo "🟩 Testing TypeScript CommonJS" 57 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 58 | cd ts-cjs-project 59 | npm install 60 | npm install "$ROOT_DIR" 61 | npm test 62 | cd .. 63 | echo "" 64 | 65 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 66 | echo "✨ ALL COMPATIBILITY TESTS PASSED!" 67 | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" 68 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "iceberg-js", 3 | "version": "0.8.1", 4 | "description": "A small, framework-agnostic JavaScript/TypeScript client for the Apache Iceberg REST Catalog", 5 | "type": "module", 6 | "main": "./dist/index.cjs", 7 | "module": "./dist/index.mjs", 8 | "types": "./dist/index.d.ts", 9 | "exports": { 10 | ".": { 11 | "import": { 12 | "types": "./dist/index.d.ts", 13 | "default": "./dist/index.mjs" 14 | }, 15 | "require": { 16 | "types": "./dist/index.d.cts", 17 | "default": "./dist/index.cjs" 18 | }, 19 | "default": "./dist/index.mjs" 20 | } 21 | }, 22 | "files": [ 23 | "dist" 24 | ], 25 | "scripts": { 26 | "build": "tsup", 27 | "dev": "tsup --watch", 28 | "docs": "typedoc src/index.ts", 29 | "format": "prettier --write .", 30 | "lint": "eslint .", 31 | "type-check": "tsc --noEmit", 32 | "test": "vitest run", 33 | "test:watch": "vitest watch", 34 | "test:integration": "bash scripts/test-integration.sh", 35 | "test:integration:ci": "bash scripts/test-integration.sh --cleanup", 36 | "test:compatibility": "bash test/compatibility/run-all.sh", 37 | "check": "pnpm lint && pnpm type-check && pnpm test && pnpm build" 38 | }, 39 | "keywords": [ 40 | "iceberg", 41 | "apache-iceberg", 42 | "rest-catalog", 43 | "data-lake", 44 | "catalog" 45 | ], 46 | "author": "mandarini", 47 | "license": "MIT", 48 | "repository": { 49 | "type": "git", 50 | "url": "https://github.com/supabase/iceberg-js" 51 | }, 52 | "bugs": { 53 | "url": "https://github.com/supabase/iceberg-js/issues" 54 | }, 55 | "homepage": "https://supabase.github.io/iceberg-js/", 56 | "publishConfig": { 57 | "access": "public" 58 | }, 59 | "devDependencies": { 60 | "@eslint/js": "^9.39.1", 61 | "@eslint/json": "^0.14.0", 62 | "@eslint/markdown": "^7.5.1", 63 | "@types/node": "^20.0.0", 64 | "eslint": "^9.39.1", 65 | "globals": "^16.5.0", 66 | "jiti": "^2.6.1", 67 | "prettier": "^3.6.2", 68 | "tsup": "^8.5.1", 69 | "typedoc": "^0.28.14", 70 | "typescript": "^5.9.3", 71 | "typescript-eslint": "^8.47.0", 72 | "vitest": "^4.0.12" 73 | }, 74 | "engines": { 75 | "node": ">=20.0.0" 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/http/createFetchClient.ts: -------------------------------------------------------------------------------- 1 | import { IcebergError, type IcebergErrorResponse } from '../errors/IcebergError' 2 | import { buildUrl } from '../utils/url' 3 | import type { AuthConfig, HttpClient, HttpRequest, HttpResponse } from './types' 4 | 5 | async function buildAuthHeaders(auth?: AuthConfig): Promise> { 6 | if (!auth || auth.type === 'none') { 7 | return {} 8 | } 9 | 10 | if (auth.type === 'bearer') { 11 | return { Authorization: `Bearer ${auth.token}` } 12 | } 13 | 14 | if (auth.type === 'header') { 15 | return { [auth.name]: auth.value } 16 | } 17 | 18 | if (auth.type === 'custom') { 19 | return await auth.getHeaders() 20 | } 21 | 22 | return {} 23 | } 24 | 25 | export function createFetchClient(options: { 26 | baseUrl: string 27 | auth?: AuthConfig 28 | fetchImpl?: typeof fetch 29 | }): HttpClient { 30 | const fetchFn = options.fetchImpl ?? globalThis.fetch 31 | 32 | return { 33 | async request({ 34 | method, 35 | path, 36 | query, 37 | body, 38 | headers, 39 | }: HttpRequest): Promise> { 40 | const url = buildUrl(options.baseUrl, path, query) 41 | const authHeaders = await buildAuthHeaders(options.auth) 42 | 43 | const res = await fetchFn(url, { 44 | method, 45 | headers: { 46 | ...(body ? { 'Content-Type': 'application/json' } : {}), 47 | ...authHeaders, 48 | ...headers, 49 | }, 50 | body: body ? JSON.stringify(body) : undefined, 51 | }) 52 | 53 | const text = await res.text() 54 | const isJson = (res.headers.get('content-type') || '').includes('application/json') 55 | const data = isJson && text ? (JSON.parse(text) as T) : (text as T) 56 | 57 | if (!res.ok) { 58 | const errBody = isJson ? (data as IcebergErrorResponse) : undefined 59 | const errorDetail = errBody?.error 60 | throw new IcebergError(errorDetail?.message ?? `Request failed with status ${res.status}`, { 61 | status: res.status, 62 | icebergType: errorDetail?.type, 63 | icebergCode: errorDetail?.code, 64 | details: errBody, 65 | }) 66 | } 67 | 68 | return { status: res.status, headers: res.headers, data: data as T } 69 | }, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /examples/basic-usage.ts: -------------------------------------------------------------------------------- 1 | import { IcebergRestCatalog, getCurrentSchema } from '../src/index' 2 | 3 | async function main() { 4 | const catalog = new IcebergRestCatalog({ 5 | baseUrl: 'https://my-catalog.example.com/iceberg/v1', 6 | auth: { 7 | type: 'bearer', 8 | token: 'your-token-here', 9 | }, 10 | }) 11 | 12 | try { 13 | // List namespaces 14 | const namespaces = await catalog.listNamespaces() 15 | console.log('Namespaces:', namespaces) 16 | 17 | // Create namespace 18 | await catalog.createNamespace( 19 | { namespace: ['analytics'] }, 20 | { properties: { owner: 'data-team' } } 21 | ) 22 | 23 | // List tables 24 | const tables = await catalog.listTables({ namespace: ['analytics'] }) 25 | console.log('Tables:', tables) 26 | 27 | // Create table 28 | const metadata = await catalog.createTable( 29 | { namespace: ['analytics'] }, 30 | { 31 | name: 'events', 32 | schema: { 33 | type: 'struct', 34 | fields: [ 35 | { id: 1, name: 'id', type: 'long', required: true }, 36 | { id: 2, name: 'timestamp', type: 'timestamp', required: true }, 37 | { id: 3, name: 'user_id', type: 'string', required: false }, 38 | ], 39 | 'schema-id': 0, 40 | 'identifier-field-ids': [1], 41 | }, 42 | 'partition-spec': { 43 | 'spec-id': 0, 44 | fields: [ 45 | { 46 | source_id: 2, 47 | field_id: 1000, 48 | name: 'ts_day', 49 | transform: 'day', 50 | }, 51 | ], 52 | }, 53 | 'write-order': { 54 | 'order-id': 0, 55 | fields: [], 56 | }, 57 | properties: { 58 | 'write.format.default': 'parquet', 59 | }, 60 | } 61 | ) 62 | 63 | console.log('Created table:', metadata.name) 64 | 65 | // Load table 66 | const loadedTable = await catalog.loadTable({ 67 | namespace: ['analytics'], 68 | name: 'events', 69 | }) 70 | console.log('Loaded table location:', loadedTable.location) 71 | 72 | // Get the current schema 73 | const schema = getCurrentSchema(loadedTable) 74 | console.log('Current schema:', schema) 75 | } catch (error) { 76 | console.error('Error:', error) 77 | } 78 | } 79 | 80 | main() 81 | -------------------------------------------------------------------------------- /scripts/test-integration.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # Colors for output 6 | RED='\033[0;31m' 7 | GREEN='\033[0;32m' 8 | YELLOW='\033[1;33m' 9 | NC='\033[0m' # No Color 10 | 11 | # Default values 12 | CLEANUP=${CLEANUP:-false} 13 | WAIT_TIMEOUT=${WAIT_TIMEOUT:-60} 14 | 15 | # Parse command line arguments 16 | while [[ $# -gt 0 ]]; do 17 | case $1 in 18 | --cleanup) 19 | CLEANUP=true 20 | shift 21 | ;; 22 | --wait-timeout) 23 | WAIT_TIMEOUT="$2" 24 | shift 2 25 | ;; 26 | *) 27 | echo "Unknown option: $1" 28 | echo "Usage: $0 [--cleanup] [--wait-timeout SECONDS]" 29 | exit 1 30 | ;; 31 | esac 32 | done 33 | 34 | echo -e "${GREEN}Starting integration test...${NC}\n" 35 | 36 | # Function to cleanup on exit 37 | cleanup() { 38 | if [ "$CLEANUP" = true ]; then 39 | echo -e "\n${YELLOW}Cleaning up Docker containers...${NC}" 40 | docker compose down -v 41 | echo -e "${GREEN}Cleanup complete${NC}" 42 | else 43 | echo -e "\n${YELLOW}Docker containers are still running. Use 'docker compose down -v' to stop them.${NC}" 44 | fi 45 | } 46 | 47 | # Set trap to cleanup on exit 48 | trap cleanup EXIT 49 | 50 | # Start docker compose services 51 | echo -e "${GREEN}Starting Docker services...${NC}" 52 | docker compose up -d 53 | 54 | # Wait for iceberg-rest to be healthy 55 | echo -e "${GREEN}Waiting for Iceberg REST Catalog to be ready...${NC}" 56 | SECONDS_WAITED=0 57 | until curl -s http://localhost:8181/v1/config > /dev/null 2>&1; do 58 | if [ $SECONDS_WAITED -ge $WAIT_TIMEOUT ]; then 59 | echo -e "${RED}Timeout waiting for Iceberg REST Catalog to start${NC}" 60 | docker compose logs iceberg-rest 61 | exit 1 62 | fi 63 | echo "Waiting for catalog... ($SECONDS_WAITED/$WAIT_TIMEOUT seconds)" 64 | sleep 2 65 | SECONDS_WAITED=$((SECONDS_WAITED + 2)) 66 | done 67 | 68 | echo -e "${GREEN}Iceberg REST Catalog is ready!${NC}\n" 69 | 70 | # Wait a bit more for MinIO to be fully ready 71 | echo -e "${GREEN}Waiting for MinIO to be ready...${NC}" 72 | sleep 3 73 | 74 | # Run the integration test 75 | echo -e "${GREEN}Running integration test...${NC}\n" 76 | npx vitest run --config vitest.config.integration.ts 77 | 78 | TEST_EXIT_CODE=$? 79 | 80 | if [ $TEST_EXIT_CODE -eq 0 ]; then 81 | echo -e "\n${GREEN}✨ Integration tests passed!${NC}" 82 | else 83 | echo -e "\n${RED}❌ Integration tests failed${NC}" 84 | exit $TEST_EXIT_CODE 85 | fi 86 | 87 | exit 0 88 | -------------------------------------------------------------------------------- /src/catalog/namespaces.ts: -------------------------------------------------------------------------------- 1 | import type { HttpClient } from '../http/types' 2 | import { IcebergError } from '../errors/IcebergError' 3 | import type { 4 | CreateNamespaceRequest, 5 | CreateNamespaceResponse, 6 | GetNamespaceResponse, 7 | ListNamespacesResponse, 8 | NamespaceIdentifier, 9 | NamespaceMetadata, 10 | } from './types' 11 | 12 | function namespaceToPath(namespace: string[]): string { 13 | return namespace.join('\x1F') 14 | } 15 | 16 | export class NamespaceOperations { 17 | constructor( 18 | private readonly client: HttpClient, 19 | private readonly prefix: string = '' 20 | ) {} 21 | 22 | async listNamespaces(parent?: NamespaceIdentifier): Promise { 23 | const query = parent ? { parent: namespaceToPath(parent.namespace) } : undefined 24 | 25 | const response = await this.client.request({ 26 | method: 'GET', 27 | path: `${this.prefix}/namespaces`, 28 | query, 29 | }) 30 | 31 | return response.data.namespaces.map((ns) => ({ namespace: ns })) 32 | } 33 | 34 | async createNamespace( 35 | id: NamespaceIdentifier, 36 | metadata?: NamespaceMetadata 37 | ): Promise { 38 | const request: CreateNamespaceRequest = { 39 | namespace: id.namespace, 40 | properties: metadata?.properties, 41 | } 42 | 43 | const response = await this.client.request({ 44 | method: 'POST', 45 | path: `${this.prefix}/namespaces`, 46 | body: request, 47 | }) 48 | 49 | return response.data 50 | } 51 | 52 | async dropNamespace(id: NamespaceIdentifier): Promise { 53 | await this.client.request({ 54 | method: 'DELETE', 55 | path: `${this.prefix}/namespaces/${namespaceToPath(id.namespace)}`, 56 | }) 57 | } 58 | 59 | async loadNamespaceMetadata(id: NamespaceIdentifier): Promise { 60 | const response = await this.client.request({ 61 | method: 'GET', 62 | path: `${this.prefix}/namespaces/${namespaceToPath(id.namespace)}`, 63 | }) 64 | 65 | return { 66 | properties: response.data.properties, 67 | } 68 | } 69 | 70 | async namespaceExists(id: NamespaceIdentifier): Promise { 71 | try { 72 | await this.client.request({ 73 | method: 'HEAD', 74 | path: `${this.prefix}/namespaces/${namespaceToPath(id.namespace)}`, 75 | }) 76 | return true 77 | } catch (error) { 78 | if (error instanceof IcebergError && error.status === 404) { 79 | return false 80 | } 81 | throw error 82 | } 83 | } 84 | 85 | async createNamespaceIfNotExists( 86 | id: NamespaceIdentifier, 87 | metadata?: NamespaceMetadata 88 | ): Promise { 89 | try { 90 | return await this.createNamespace(id, metadata) 91 | } catch (error) { 92 | if (error instanceof IcebergError && error.status === 409) { 93 | return 94 | } 95 | throw error 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /test/integration/TESTING-DOCKER.md: -------------------------------------------------------------------------------- 1 | # Testing iceberg-js with Local Docker Catalog 2 | 3 | This guide shows you how to test `iceberg-js` against a local Iceberg REST Catalog running in Docker. 4 | 5 | ## Prerequisites 6 | 7 | - Docker and Docker Compose installed 8 | - Node.js 20+ and pnpm 9 | 10 | ## Quick Start 11 | 12 | ### 1. Run integration tests 13 | 14 | The easiest way to run integration tests is using the provided script: 15 | 16 | ```bash 17 | pnpm test:integration 18 | ``` 19 | 20 | This will: 21 | 22 | - Start Docker services (Iceberg REST Catalog + MinIO) 23 | - Wait for services to be ready 24 | - Run the integration test 25 | - Leave containers running for debugging 26 | 27 | To automatically clean up containers after the test: 28 | 29 | ```bash 30 | pnpm test:integration:ci 31 | ``` 32 | 33 | ### 2. Manual approach 34 | 35 | If you want more control: 36 | 37 | ```bash 38 | # Start the Iceberg REST Catalog 39 | docker compose up -d 40 | 41 | # Verify it's running 42 | curl http://localhost:8181/v1/config 43 | 44 | # Run the test script 45 | npx tsx test/integration/test-local-catalog.ts 46 | 47 | # Stop containers when done 48 | docker compose down -v 49 | ``` 50 | 51 | This starts: 52 | 53 | - **Iceberg REST Catalog** on `http://localhost:8181` 54 | - **MinIO** (S3-compatible storage) on `http://localhost:9000` (API) and `http://localhost:9001` (Console) 55 | 56 | ## Accessing MinIO Console 57 | 58 | You can view the underlying S3 storage: 59 | 60 | 1. Open http://localhost:9001 in your browser 61 | 2. Login with: 62 | - Username: `supa-storage` 63 | - Password: `secret1234` 64 | 3. Navigate to the `warehouse--table-s3` bucket to see table data 65 | 66 | ## Stopping the Catalog 67 | 68 | ```bash 69 | # Stop containers but keep data 70 | docker compose stop 71 | 72 | # Stop and remove containers + data 73 | docker compose down -v 74 | ``` 75 | 76 | ## Troubleshooting 77 | 78 | ### Port already in use 79 | 80 | If port 8181 or 9000 is already taken, edit `docker-compose.yml` and change the ports: 81 | 82 | ```yaml 83 | ports: 84 | - '8182:8181' # Use 8182 instead of 8181 85 | ``` 86 | 87 | Then update `test/integration/test-local-catalog.ts` to use the new port. 88 | 89 | ### Catalog not responding 90 | 91 | Wait a few seconds after `docker-compose up` for the services to fully start: 92 | 93 | ```bash 94 | # Check logs 95 | docker compose logs -f iceberg-rest 96 | 97 | # Wait for "Started ServerConnector" message 98 | ``` 99 | 100 | ### Integration test script options 101 | 102 | The integration test script supports options: 103 | 104 | ```bash 105 | # Run with custom wait timeout (default 60 seconds) 106 | bash scripts/test-integration.sh --wait-timeout 120 107 | 108 | # Run with cleanup 109 | bash scripts/test-integration.sh --cleanup 110 | ``` 111 | 112 | ## What's Running? 113 | 114 | - **Iceberg REST Catalog** (`tabulario/iceberg-rest`): Official Apache Iceberg REST catalog implementation 115 | - **MinIO**: S3-compatible object storage where table data and metadata are stored 116 | 117 | This setup matches the configuration used in the Supabase Storage repository for consistency. 118 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [0.8.1](https://github.com/supabase/iceberg-js/compare/iceberg-js-v0.8.0...iceberg-js-v0.8.1) (2025-11-28) 4 | 5 | ### Bug Fixes 6 | 7 | - align IcebergType with OpenAPI spec ([#24](https://github.com/supabase/iceberg-js/issues/24)) ([82ed555](https://github.com/supabase/iceberg-js/commit/82ed555f4d9fe2bb9e26736590c87bec3b0f19f0)) 8 | 9 | ## [0.8.0](https://github.com/supabase/iceberg-js/compare/iceberg-js-v0.7.0...iceberg-js-v0.8.0) (2025-11-25) 10 | 11 | ### Features 12 | 13 | - match Catalog API return types and error structure ([#22](https://github.com/supabase/iceberg-js/issues/22)) ([87dbaba](https://github.com/supabase/iceberg-js/commit/87dbabac6e8c4cf66183aaf79109722a1339c95c)) 14 | 15 | ## [0.7.0](https://github.com/supabase/iceberg-js/compare/iceberg-js-v0.6.0...iceberg-js-v0.7.0) (2025-11-21) 16 | 17 | ### Features 18 | 19 | - add namespace and table existence checks with creation methods ([#18](https://github.com/supabase/iceberg-js/issues/18)) ([7a5d0e5](https://github.com/supabase/iceberg-js/commit/7a5d0e55d62898557c5dd69e39e8486385e87b11)) 20 | 21 | ## [0.6.0](https://github.com/supabase/iceberg-js/compare/iceberg-js-v0.5.1...iceberg-js-v0.6.0) (2025-11-21) 22 | 23 | ### Features 24 | 25 | - add purge flag to dropTable method ([#14](https://github.com/supabase/iceberg-js/issues/14)) ([ba5b723](https://github.com/supabase/iceberg-js/commit/ba5b7232a593b0240f6b65de5308e1ce7bf3df40)) 26 | - release for all compatible envs ([#15](https://github.com/supabase/iceberg-js/issues/15)) ([5f649f8](https://github.com/supabase/iceberg-js/commit/5f649f85b9930203d20580edf77b6a44c0dd545b)) 27 | 28 | ### Bug Fixes 29 | 30 | - ensure baseUrl has trailing / for proper URL creation ([#13](https://github.com/supabase/iceberg-js/issues/13)) ([6baf4b9](https://github.com/supabase/iceberg-js/commit/6baf4b9fa3252dc0c98c46a1d8c5f76a316a5071)) 31 | 32 | ## [0.5.1](https://github.com/supabase/iceberg-js/compare/iceberg-js-v0.5.0...iceberg-js-v0.5.1) (2025-11-21) 33 | 34 | ### Bug Fixes 35 | 36 | - format all files ([#11](https://github.com/supabase/iceberg-js/issues/11)) ([f95874b](https://github.com/supabase/iceberg-js/commit/f95874b3b94bb7fff32970ebebbbeb7745263e02)) 37 | - update npm to latest for release ([#10](https://github.com/supabase/iceberg-js/issues/10)) ([7943aca](https://github.com/supabase/iceberg-js/commit/7943acab02005ad5e5be249a201fe2dce810b57b)) 38 | 39 | ## [0.5.0](https://github.com/supabase/iceberg-js/compare/iceberg-js-v0.4.0...iceberg-js-v0.5.0) (2025-11-21) 40 | 41 | ### Features 42 | 43 | - accessDelegation and codeowners ([94c65c5](https://github.com/supabase/iceberg-js/commit/94c65c503949f67d03fb7d5fe1cafd90d2196fcd)) 44 | - release ([#4](https://github.com/supabase/iceberg-js/issues/4)) ([8b0dfbc](https://github.com/supabase/iceberg-js/commit/8b0dfbc711e4db4a4c311ec5f3c4532bd808d1e3)) 45 | 46 | ### Bug Fixes 47 | 48 | - manual release workflow ([#6](https://github.com/supabase/iceberg-js/issues/6)) ([f00cea5](https://github.com/supabase/iceberg-js/commit/f00cea517fc7a57f54f868f429554c3480792ba0)) 49 | - update prefix to include version before catalog name ([#8](https://github.com/supabase/iceberg-js/issues/8)) ([db4f2ca](https://github.com/supabase/iceberg-js/commit/db4f2ca6e99469108592d66ea4a9abcc14b8bbad)) 50 | -------------------------------------------------------------------------------- /examples/supabase-storage-usage.ts: -------------------------------------------------------------------------------- 1 | import { IcebergRestCatalog } from '../src/index' 2 | 3 | /** 4 | * Example: Using iceberg-js with Supabase Storage Analytics 5 | * 6 | * This demonstrates how to create tables in Supabase's Iceberg REST Catalog 7 | * using the iceberg-js library. 8 | */ 9 | async function main() { 10 | // Initialize the catalog client with Supabase Storage endpoint 11 | const catalog = new IcebergRestCatalog({ 12 | baseUrl: 'your-catalog-url', 13 | auth: { 14 | type: 'bearer', 15 | token: process.env.SUPABASE_TOKEN || 'your-token-here', 16 | }, 17 | // Request vended credentials from Supabase Storage 18 | // This allows the server to provide temporary AWS credentials 19 | // for accessing table data files in S3 20 | accessDelegation: ['vended-credentials'], 21 | }) 22 | 23 | try { 24 | // Ensure the default namespace exists 25 | try { 26 | await catalog.createNamespace( 27 | { namespace: ['default'] }, 28 | { properties: { description: 'Default namespace for analytics tables' } } 29 | ) 30 | console.log('✓ Created default namespace') 31 | } catch (err) { 32 | if (err instanceof Error && err.message.includes('already exists')) { 33 | console.log('✓ Default namespace already exists') 34 | } else { 35 | throw err 36 | } 37 | } 38 | 39 | // Create an analytics table 40 | const tableMetadata = await catalog.createTable( 41 | { namespace: ['default'] }, 42 | { 43 | name: 'analytics_events', 44 | schema: { 45 | type: 'struct', 46 | fields: [ 47 | { id: 1, name: 'id', type: 'long', required: true }, 48 | { id: 2, name: 'event_name', type: 'string', required: true }, 49 | { id: 3, name: 'timestamp', type: 'timestamp', required: true }, 50 | { id: 4, name: 'user_id', type: 'string', required: false }, 51 | { id: 5, name: 'properties', type: 'string', required: false }, 52 | ], 53 | 'schema-id': 0, 54 | 'identifier-field-ids': [1], 55 | }, 56 | 'partition-spec': { 57 | 'spec-id': 0, 58 | fields: [ 59 | { 60 | source_id: 3, // timestamp field 61 | field_id: 1000, 62 | name: 'event_day', 63 | transform: 'day', 64 | }, 65 | ], 66 | }, 67 | 'write-order': { 68 | 'order-id': 0, 69 | fields: [], 70 | }, 71 | properties: { 72 | 'write.format.default': 'parquet', 73 | 'write.parquet.compression-codec': 'snappy', 74 | }, 75 | } 76 | ) 77 | 78 | console.log('✓ Created table:', tableMetadata.name) 79 | console.log(' Location:', tableMetadata.location) 80 | console.log(' Schema ID:', tableMetadata['current-schema-id']) 81 | 82 | // List all tables in the namespace 83 | const tables = await catalog.listTables({ namespace: ['default'] }) 84 | console.log( 85 | '✓ Tables in default namespace:', 86 | tables.map((t) => t.name) 87 | ) 88 | 89 | // Load the table metadata 90 | const loadedTable = await catalog.loadTable({ 91 | namespace: ['default'], 92 | name: 'analytics_events', 93 | }) 94 | console.log('✓ Loaded table location:', loadedTable.location) 95 | } catch (error) { 96 | console.error('Error:', error) 97 | process.exit(1) 98 | } 99 | } 100 | 101 | main() 102 | -------------------------------------------------------------------------------- /src/catalog/tables.ts: -------------------------------------------------------------------------------- 1 | import type { HttpClient } from '../http/types' 2 | import { IcebergError } from '../errors/IcebergError' 3 | import type { 4 | CreateTableRequest, 5 | CommitTableResponse, 6 | ListTablesResponse, 7 | LoadTableResponse, 8 | NamespaceIdentifier, 9 | TableIdentifier, 10 | TableMetadata, 11 | UpdateTableRequest, 12 | DropTableRequest, 13 | } from './types' 14 | 15 | function namespaceToPath(namespace: string[]): string { 16 | return namespace.join('\x1F') 17 | } 18 | 19 | export class TableOperations { 20 | constructor( 21 | private readonly client: HttpClient, 22 | private readonly prefix: string = '', 23 | private readonly accessDelegation?: string 24 | ) {} 25 | 26 | async listTables(namespace: NamespaceIdentifier): Promise { 27 | const response = await this.client.request({ 28 | method: 'GET', 29 | path: `${this.prefix}/namespaces/${namespaceToPath(namespace.namespace)}/tables`, 30 | }) 31 | 32 | return response.data.identifiers 33 | } 34 | 35 | async createTable( 36 | namespace: NamespaceIdentifier, 37 | request: CreateTableRequest 38 | ): Promise { 39 | const headers: Record = {} 40 | if (this.accessDelegation) { 41 | headers['X-Iceberg-Access-Delegation'] = this.accessDelegation 42 | } 43 | 44 | const response = await this.client.request({ 45 | method: 'POST', 46 | path: `${this.prefix}/namespaces/${namespaceToPath(namespace.namespace)}/tables`, 47 | body: request, 48 | headers, 49 | }) 50 | 51 | return response.data.metadata 52 | } 53 | 54 | async updateTable( 55 | id: TableIdentifier, 56 | request: UpdateTableRequest 57 | ): Promise { 58 | const response = await this.client.request({ 59 | method: 'POST', 60 | path: `${this.prefix}/namespaces/${namespaceToPath(id.namespace)}/tables/${id.name}`, 61 | body: request, 62 | }) 63 | 64 | return { 65 | 'metadata-location': response.data['metadata-location'], 66 | metadata: response.data.metadata, 67 | } 68 | } 69 | 70 | async dropTable(id: TableIdentifier, options?: DropTableRequest): Promise { 71 | await this.client.request({ 72 | method: 'DELETE', 73 | path: `${this.prefix}/namespaces/${namespaceToPath(id.namespace)}/tables/${id.name}`, 74 | query: { purgeRequested: String(options?.purge ?? false) }, 75 | }) 76 | } 77 | 78 | async loadTable(id: TableIdentifier): Promise { 79 | const headers: Record = {} 80 | if (this.accessDelegation) { 81 | headers['X-Iceberg-Access-Delegation'] = this.accessDelegation 82 | } 83 | 84 | const response = await this.client.request({ 85 | method: 'GET', 86 | path: `${this.prefix}/namespaces/${namespaceToPath(id.namespace)}/tables/${id.name}`, 87 | headers, 88 | }) 89 | 90 | return response.data.metadata 91 | } 92 | 93 | async tableExists(id: TableIdentifier): Promise { 94 | const headers: Record = {} 95 | if (this.accessDelegation) { 96 | headers['X-Iceberg-Access-Delegation'] = this.accessDelegation 97 | } 98 | 99 | try { 100 | await this.client.request({ 101 | method: 'HEAD', 102 | path: `${this.prefix}/namespaces/${namespaceToPath(id.namespace)}/tables/${id.name}`, 103 | headers, 104 | }) 105 | return true 106 | } catch (error) { 107 | if (error instanceof IcebergError && error.status === 404) { 108 | return false 109 | } 110 | throw error 111 | } 112 | } 113 | 114 | async createTableIfNotExists( 115 | namespace: NamespaceIdentifier, 116 | request: CreateTableRequest 117 | ): Promise { 118 | try { 119 | return await this.createTable(namespace, request) 120 | } catch (error) { 121 | if (error instanceof IcebergError && error.status === 409) { 122 | return await this.loadTable({ namespace: namespace.namespace, name: request.name }) 123 | } 124 | throw error 125 | } 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/catalog/types.ts: -------------------------------------------------------------------------------- 1 | export interface NamespaceIdentifier { 2 | namespace: string[] 3 | } 4 | 5 | export interface NamespaceMetadata { 6 | properties: Record 7 | } 8 | 9 | export interface TableIdentifier { 10 | namespace: string[] 11 | name: string 12 | } 13 | 14 | /** 15 | * Primitive types in Iceberg - all represented as strings. 16 | * Parameterized types use string format: decimal(precision,scale) and fixed[length] 17 | * 18 | * Note: The OpenAPI spec defines PrimitiveType as `type: string`, so any string is valid. 19 | * We include known types for autocomplete, plus a catch-all for flexibility. 20 | */ 21 | export type PrimitiveType = 22 | | 'boolean' 23 | | 'int' 24 | | 'long' 25 | | 'float' 26 | | 'double' 27 | | 'string' 28 | | 'timestamp' 29 | | 'date' 30 | | 'time' 31 | | 'timestamptz' 32 | | 'uuid' 33 | | 'binary' 34 | | `decimal(${number},${number})` 35 | | `fixed[${number}]` 36 | | (string & {}) // catch-all for any format (e.g., "decimal(10, 2)" with spaces) and future types 37 | 38 | /** 39 | * Regex patterns for parsing parameterized types. 40 | * These allow flexible whitespace matching. 41 | */ 42 | const DECIMAL_REGEX = /^decimal\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)$/ 43 | const FIXED_REGEX = /^fixed\s*\[\s*(\d+)\s*\]$/ 44 | 45 | /** 46 | * Parse a decimal type string into its components. 47 | * Handles any whitespace formatting (e.g., "decimal(10,2)", "decimal(10, 2)", "decimal( 10 , 2 )"). 48 | * 49 | * @param type - The type string to parse 50 | * @returns Object with precision and scale, or null if not a valid decimal type 51 | */ 52 | export function parseDecimalType(type: string): { precision: number; scale: number } | null { 53 | const match = type.match(DECIMAL_REGEX) 54 | if (!match) return null 55 | return { 56 | precision: parseInt(match[1], 10), 57 | scale: parseInt(match[2], 10), 58 | } 59 | } 60 | 61 | /** 62 | * Parse a fixed type string into its length. 63 | * Handles any whitespace formatting (e.g., "fixed[16]", "fixed[ 16 ]"). 64 | * 65 | * @param type - The type string to parse 66 | * @returns Object with length, or null if not a valid fixed type 67 | */ 68 | export function parseFixedType(type: string): { length: number } | null { 69 | const match = type.match(FIXED_REGEX) 70 | if (!match) return null 71 | return { 72 | length: parseInt(match[1], 10), 73 | } 74 | } 75 | 76 | /** 77 | * Check if a type string is a decimal type. 78 | */ 79 | export function isDecimalType(type: string): boolean { 80 | return DECIMAL_REGEX.test(type) 81 | } 82 | 83 | /** 84 | * Check if a type string is a fixed type. 85 | */ 86 | export function isFixedType(type: string): boolean { 87 | return FIXED_REGEX.test(type) 88 | } 89 | 90 | /** 91 | * Compare two Iceberg type strings for equality, ignoring whitespace differences. 92 | * This is useful when comparing types from user input vs catalog responses, 93 | * as catalogs may normalize whitespace differently. 94 | * 95 | * @param a - First type string 96 | * @param b - Second type string 97 | * @returns true if the types are equivalent 98 | */ 99 | export function typesEqual(a: string, b: string): boolean { 100 | // For decimal types, compare parsed values 101 | const decimalA = parseDecimalType(a) 102 | const decimalB = parseDecimalType(b) 103 | if (decimalA && decimalB) { 104 | return decimalA.precision === decimalB.precision && decimalA.scale === decimalB.scale 105 | } 106 | 107 | // For fixed types, compare parsed values 108 | const fixedA = parseFixedType(a) 109 | const fixedB = parseFixedType(b) 110 | if (fixedA && fixedB) { 111 | return fixedA.length === fixedB.length 112 | } 113 | 114 | // For other types, direct string comparison 115 | return a === b 116 | } 117 | 118 | /** 119 | * Struct type - a nested structure containing fields. 120 | * Used for nested records within a field. 121 | */ 122 | export interface StructType { 123 | type: 'struct' 124 | fields: StructField[] 125 | } 126 | 127 | /** 128 | * List type - an array of elements. 129 | */ 130 | export interface ListType { 131 | type: 'list' 132 | 'element-id': number 133 | element: IcebergType 134 | 'element-required': boolean 135 | } 136 | 137 | /** 138 | * Map type - a key-value mapping. 139 | */ 140 | export interface MapType { 141 | type: 'map' 142 | 'key-id': number 143 | key: IcebergType 144 | 'value-id': number 145 | value: IcebergType 146 | 'value-required': boolean 147 | } 148 | 149 | /** 150 | * Union of all Iceberg types. 151 | * Can be a primitive type (string) or a complex type (struct, list, map). 152 | */ 153 | export type IcebergType = PrimitiveType | StructType | ListType | MapType 154 | 155 | /** 156 | * Primitive type values for default values. 157 | * Represents the possible values for initial-default and write-default. 158 | */ 159 | export type PrimitiveTypeValue = boolean | number | string 160 | 161 | /** 162 | * A field within a struct (used in nested StructType). 163 | */ 164 | export interface StructField { 165 | id: number 166 | name: string 167 | type: IcebergType 168 | required: boolean 169 | doc?: string 170 | 'initial-default'?: PrimitiveTypeValue 171 | 'write-default'?: PrimitiveTypeValue 172 | } 173 | 174 | /** 175 | * A field within a table schema (top-level). 176 | * Equivalent to StructField but kept for backwards compatibility. 177 | */ 178 | export interface TableField { 179 | id: number 180 | name: string 181 | type: IcebergType 182 | required: boolean 183 | doc?: string 184 | 'initial-default'?: PrimitiveTypeValue 185 | 'write-default'?: PrimitiveTypeValue 186 | } 187 | 188 | export interface TableSchema { 189 | type: 'struct' 190 | fields: TableField[] 191 | 'schema-id'?: number 192 | 'identifier-field-ids'?: number[] 193 | } 194 | 195 | export interface PartitionField { 196 | source_id: number 197 | field_id: number 198 | name: string 199 | transform: string 200 | } 201 | 202 | export interface PartitionSpec { 203 | 'spec-id': number 204 | fields: PartitionField[] 205 | } 206 | 207 | export interface SortField { 208 | source_id: number 209 | transform: string 210 | direction: 'asc' | 'desc' 211 | null_order: 'nulls-first' | 'nulls-last' 212 | } 213 | 214 | export interface SortOrder { 215 | 'order-id': number 216 | fields: SortField[] 217 | } 218 | 219 | export interface CreateTableRequest { 220 | name: string 221 | schema: TableSchema 222 | 'partition-spec'?: PartitionSpec 223 | 'write-order'?: SortOrder 224 | properties?: Record 225 | 'stage-create'?: boolean 226 | } 227 | 228 | export interface UpdateTableRequest { 229 | schema?: TableSchema 230 | 'partition-spec'?: PartitionSpec 231 | properties?: Record 232 | } 233 | 234 | export interface DropTableRequest { 235 | purge?: boolean 236 | } 237 | 238 | export interface TableMetadata { 239 | name?: string 240 | location: string 241 | schemas: TableSchema[] 242 | 'current-schema-id': number 243 | 'partition-specs': PartitionSpec[] 244 | 'default-spec-id'?: number 245 | 'sort-orders': SortOrder[] 246 | 'default-sort-order-id'?: number 247 | properties: Record 248 | 'metadata-location'?: string 249 | 'current-snapshot-id'?: number 250 | snapshots?: unknown[] 251 | 'snapshot-log'?: unknown[] 252 | 'metadata-log'?: unknown[] 253 | refs?: Record 254 | 'last-updated-ms'?: number 255 | 'last-column-id'?: number 256 | 'last-sequence-number'?: number 257 | 'table-uuid'?: string 258 | 'format-version'?: number 259 | 'last-partition-id'?: number 260 | } 261 | 262 | export interface CreateNamespaceRequest { 263 | namespace: string[] 264 | properties?: Record 265 | } 266 | 267 | export interface CreateNamespaceResponse { 268 | namespace: string[] 269 | properties?: Record 270 | } 271 | 272 | export interface GetNamespaceResponse { 273 | namespace: string[] 274 | properties: Record 275 | } 276 | 277 | export interface ListNamespacesResponse { 278 | namespaces: string[][] 279 | 'next-page-token'?: string 280 | } 281 | 282 | export interface ListTablesResponse { 283 | identifiers: TableIdentifier[] 284 | 'next-page-token'?: string 285 | } 286 | 287 | export interface LoadTableResponse { 288 | 'metadata-location': string 289 | metadata: TableMetadata 290 | config?: Record 291 | } 292 | 293 | export interface CommitTableResponse { 294 | 'metadata-location': string 295 | metadata: TableMetadata 296 | } 297 | 298 | /** 299 | * Gets the current (active) schema from table metadata. 300 | * 301 | * @param metadata - Table metadata containing schemas array and current-schema-id 302 | * @returns The current table schema, or undefined if not found 303 | */ 304 | export function getCurrentSchema(metadata: TableMetadata): TableSchema | undefined { 305 | return metadata.schemas.find((s) => s['schema-id'] === metadata['current-schema-id']) 306 | } 307 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to `iceberg-js` 2 | 3 | Thank you for your interest in contributing to `iceberg-js`! This document provides guidelines and instructions for contributing to the project. 4 | 5 | ## Table of Contents 6 | 7 | - [Getting Started](#getting-started) 8 | - [Development Setup](#development-setup) 9 | - [Project Structure](#project-structure) 10 | - [Development Workflow](#development-workflow) 11 | - [Testing](#testing) 12 | - [Code Style](#code-style) 13 | - [Submitting Changes](#submitting-changes) 14 | - [Release Process](#release-process) 15 | 16 | ## Getting Started 17 | 18 | `iceberg-js` is a TypeScript library that provides a client for the Apache Iceberg REST Catalog API. Before contributing, please: 19 | 20 | 1. Read the [README.md](./README.md) to understand the project's scope and goals 21 | 2. Read our [Code of Conduct](https://github.com/supabase/.github/blob/main/CODE_OF_CONDUCT.md) 22 | 3. Check existing [issues](https://github.com/supabase/iceberg-js/issues) and [pull requests](https://github.com/supabase/iceberg-js/pulls) to avoid duplicate work 23 | 24 | ## Development Setup 25 | 26 | ### Prerequisites 27 | 28 | - **Node.js**: 20.x or higher 29 | - **pnpm** 30 | 31 | ### Installation 32 | 33 | 1. Fork and clone the repository: 34 | 35 | ```bash 36 | git clone https://github.com/YOUR_USERNAME/iceberg-js.git 37 | cd iceberg-js 38 | ``` 39 | 40 | 2. Install dependencies: 41 | 42 | ```bash 43 | pnpm install 44 | ``` 45 | 46 | 3. Build the project to verify setup: 47 | 48 | ```bash 49 | pnpm run build 50 | ``` 51 | 52 | ## Project Structure 53 | 54 | ```text 55 | iceberg-js/ 56 | ├── src/ # Source code 57 | │ ├── catalog/ # Catalog operations 58 | │ │ ├── IcebergRestCatalog.ts # Main catalog client 59 | │ │ ├── namespaces.ts # Namespace operations 60 | │ │ ├── tables.ts # Table operations 61 | │ │ └── types.ts # TypeScript type definitions 62 | │ ├── http/ # HTTP client layer 63 | │ │ ├── HttpClient.ts # HTTP client interface 64 | │ │ ├── createFetchClient.ts # Fetch-based implementation 65 | │ │ └── types.ts # HTTP-related types 66 | │ ├── errors/ # Error handling 67 | │ │ └── IcebergError.ts # Custom error class 68 | │ ├── utils/ # Utility functions 69 | │ └── index.ts # Public API exports 70 | ├── test/ # Tests 71 | │ ├── catalog/ # Unit tests for catalog operations 72 | │ ├── http/ # Unit tests for HTTP client 73 | │ └── integration/ # Integration tests 74 | ├── scripts/ # Build and test scripts 75 | ├── dist/ # Compiled output (generated) 76 | ├── .github/workflows/ # CI/CD configuration 77 | ├── tsconfig.json # TypeScript configuration 78 | ├── tsup.config.ts # Build configuration 79 | ├── vitest.config.ts # Test configuration 80 | ├── eslint.config.ts # Linting configuration 81 | └── .prettierrc # Code formatting configuration 82 | ``` 83 | 84 | ## Development Workflow 85 | 86 | ### Building 87 | 88 | Build the library for distribution: 89 | 90 | ```bash 91 | pnpm run build 92 | ``` 93 | 94 | Watch mode for development (rebuilds on file changes): 95 | 96 | ```bash 97 | pnpm run dev 98 | ``` 99 | 100 | ### Type Checking 101 | 102 | Run TypeScript type checking without emitting files: 103 | 104 | ```bash 105 | pnpm run type-check 106 | ``` 107 | 108 | ### Formatting 109 | 110 | Format all code using Prettier: 111 | 112 | ```bash 113 | pnpm run format 114 | ``` 115 | 116 | ### Linting 117 | 118 | Run ESLint to check for code issues: 119 | 120 | ```bash 121 | pnpm run lint 122 | ``` 123 | 124 | ### Full Check 125 | 126 | Run all checks (lint, type-check, test, build): 127 | 128 | ```bash 129 | pnpm run check 130 | ``` 131 | 132 | This is what CI runs, so it's a good idea to run this before pushing. 133 | 134 | ## Testing 135 | 136 | ### Unit Tests 137 | 138 | Run unit tests with Vitest: 139 | 140 | ```bash 141 | pnpm test 142 | ``` 143 | 144 | Watch mode for test-driven development: 145 | 146 | ```bash 147 | pnpm test:watch 148 | ``` 149 | 150 | ### Integration Tests 151 | 152 | Integration tests run against a real Iceberg REST Catalog in Docker: 153 | 154 | ```bash 155 | pnpm test:integration 156 | ``` 157 | 158 | This command: 159 | 160 | 1. Starts Docker Compose services (Iceberg REST catalog + MinIO storage) 161 | 2. Runs integration tests 162 | 3. Leaves services running for debugging 163 | 164 | For CI or cleanup after testing: 165 | 166 | ```bash 167 | pnpm test:integration:ci 168 | ``` 169 | 170 | This adds automatic cleanup (stops and removes containers). 171 | 172 | ### Writing Tests 173 | 174 | - Unit tests go in `test/` with the same structure as `src/` 175 | - Use descriptive test names that explain the behavior being tested 176 | - Mock external dependencies (HTTP calls) in unit tests 177 | - Integration tests should test real catalog interactions 178 | - Follow existing test patterns for consistency 179 | 180 | ## Code Style 181 | 182 | ### TypeScript Guidelines 183 | 184 | - Use **strict TypeScript** - all code must pass type checking 185 | - Prefer **interfaces** for public APIs, **types** for unions/intersections 186 | - Export types that consumers might need 187 | - Use **explicit return types** on public methods 188 | - Avoid `any` - use `unknown` if type is truly unknown 189 | 190 | ### Naming Conventions 191 | 192 | - **Classes**: PascalCase (e.g., `IcebergRestCatalog`) 193 | - **Interfaces/Types**: PascalCase (e.g., `TableIdentifier`) 194 | - **Functions/Methods**: camelCase (e.g., `createTable`) 195 | - **Constants**: UPPER_SNAKE_CASE for true constants (e.g., `DEFAULT_TIMEOUT`) 196 | - **Private members**: prefix with underscore or use private keyword 197 | 198 | ### Documentation 199 | 200 | - All public APIs must have **TSDoc comments** 201 | - Include `@param`, `@returns`, `@throws`, and `@example` tags where appropriate 202 | - Examples in TSDoc should be runnable code 203 | - Keep documentation concise but informative 204 | 205 | Example: 206 | 207 | ````typescript 208 | /** 209 | * Creates a new table in the catalog. 210 | * 211 | * @param namespace - Namespace to create the table in 212 | * @param request - Table creation request including name, schema, partition spec 213 | * @returns Table metadata for the created table 214 | * @throws {IcebergError} If the table already exists or namespace doesn't exist 215 | * 216 | * @example 217 | * ```typescript 218 | * const metadata = await catalog.createTable( 219 | * { namespace: ['analytics'] }, 220 | * { name: 'events', schema: { ... } } 221 | * ); 222 | * ``` 223 | */ 224 | async createTable(namespace: NamespaceIdentifier, request: CreateTableRequest): Promise 225 | ```` 226 | 227 | ### Error Handling 228 | 229 | - Use `IcebergError` for all catalog-related errors 230 | - Include status code and error type from the REST API 231 | - Provide helpful error messages 232 | - Let unexpected errors propagate (don't catch everything) 233 | 234 | ## Submitting Changes 235 | 236 | ### Commit Messages 237 | 238 | We use [Conventional Commits](https://www.conventionalcommits.org/) for automated releases. Format: 239 | 240 | ```text 241 | (): 242 | 243 | [optional body] 244 | 245 | [optional footer] 246 | ``` 247 | 248 | **Types:** 249 | 250 | - `feat`: New feature (triggers minor version bump) 251 | - `fix`: Bug fix (triggers patch version bump) 252 | - `docs`: Documentation changes only 253 | - `test`: Adding or updating tests 254 | - `chore`: Maintenance tasks, dependency updates 255 | - `refactor`: Code changes that neither fix bugs nor add features 256 | - `perf`: Performance improvements 257 | - `ci`: CI/CD configuration changes 258 | 259 | **Breaking changes:** 260 | 261 | - Use `feat!:` or `fix!:` for breaking changes (triggers major version bump) 262 | - Or include `BREAKING CHANGE:` in the commit footer 263 | 264 | **Examples:** 265 | 266 | ```bash 267 | feat: add support for view operations 268 | fix: handle empty namespace list correctly 269 | feat(auth): add OAuth2 authentication support 270 | docs: update README with new examples 271 | test: add integration tests for table updates 272 | feat!: change auth config structure 273 | 274 | BREAKING CHANGE: auth configuration now uses a discriminated union 275 | ``` 276 | 277 | ### Pull Request Process 278 | 279 | 1. **Create a branch** from `main`: 280 | 281 | ```bash 282 | git checkout -b feat/my-feature 283 | ``` 284 | 285 | 2. **Make your changes** following the guidelines above 286 | 287 | 3. **Run checks** locally: 288 | 289 | ```bash 290 | pnpm run check 291 | pnpm test:integration 292 | ``` 293 | 294 | 4. **Commit** using conventional commit format: 295 | 296 | ```bash 297 | git commit -m "feat: add support for XYZ" 298 | ``` 299 | 300 | 5. **Push** to your fork: 301 | 302 | ```bash 303 | git push origin feat/my-feature 304 | ``` 305 | 306 | 6. **Open a Pull Request** with: 307 | - Clear title following conventional commit format 308 | - Description of what changed and why 309 | - Reference any related issues (e.g., "Fixes #123") 310 | - Screenshots/examples if adding user-facing features 311 | 312 | 7. **Respond to feedback** - maintainers may request changes 313 | 314 | 8. **Wait for CI** - all tests must pass before merging 315 | 316 | ### PR Guidelines 317 | 318 | - Keep PRs focused - one feature or fix per PR 319 | - Update documentation if you change public APIs 320 | - Add tests for new functionality 321 | - Ensure all CI checks pass 322 | - Rebase on `main` if needed to resolve conflicts 323 | - Be responsive to review feedback 324 | 325 | ## Release Process 326 | 327 | This project uses [release-please](https://github.com/googleapis/release-please) for automated releases. You don't need to manually manage versions or changelogs. 328 | 329 | ### How It Works 330 | 331 | 1. **You commit** using conventional commit format (see above) 332 | 333 | 2. **release-please creates/updates a release PR** automatically when changes are pushed to `main` 334 | - Updates version in `package.json` 335 | - Updates `CHANGELOG.md` 336 | - Generates release notes 337 | 338 | 3. **Maintainer merges the release PR** when ready to release 339 | - Creates a GitHub release and git tag 340 | - Automatically publishes to npm with provenance 341 | 342 | ### Version Bumps 343 | 344 | Versions follow [Semantic Versioning](https://semver.org/): 345 | 346 | - **Major (1.0.0 → 2.0.0)**: Breaking changes (`feat!:`, `fix!:`, or `BREAKING CHANGE:`) 347 | - **Minor (1.0.0 → 1.1.0)**: New features (`feat:`) 348 | - **Patch (1.0.0 → 1.0.1)**: Bug fixes (`fix:`) 349 | 350 | Commits with types like `docs:`, `test:`, `chore:` don't trigger releases on their own. 351 | 352 | ### For Maintainers Only 353 | 354 | Publishing is fully automated via GitHub Actions: 355 | 356 | 1. Merge the release-please PR when ready 357 | 2. GitHub Actions will automatically publish to npm with provenance 358 | 3. No manual `npm publish` needed 359 | 360 | ## Questions? 361 | 362 | - Open an [issue](https://github.com/supabase/iceberg-js/issues) for bugs or feature requests 363 | - Check existing issues and PRs before creating new ones 364 | - Tag your issues appropriately (`bug`, `enhancement`, `documentation`, etc.) 365 | 366 | ## License 367 | 368 | By contributing to `iceberg-js`, you agree that your contributions will be licensed under the MIT License. 369 | -------------------------------------------------------------------------------- /test/catalog/namespaces.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi } from 'vitest' 2 | import { NamespaceOperations } from '../../src/catalog/namespaces' 3 | import { IcebergError } from '../../src/errors/IcebergError' 4 | import type { HttpClient } from '../../src/http/types' 5 | 6 | describe('NamespaceOperations', () => { 7 | const createMockClient = (): HttpClient => ({ 8 | request: vi.fn(), 9 | }) 10 | 11 | describe('listNamespaces', () => { 12 | it('should list all namespaces', async () => { 13 | const mockClient = createMockClient() 14 | vi.mocked(mockClient.request).mockResolvedValue({ 15 | status: 200, 16 | headers: new Headers(), 17 | data: { 18 | namespaces: [['default'], ['analytics'], ['logs']], 19 | }, 20 | }) 21 | 22 | const ops = new NamespaceOperations(mockClient, '/v1') 23 | const result = await ops.listNamespaces() 24 | 25 | expect(result).toEqual([ 26 | { namespace: ['default'] }, 27 | { namespace: ['analytics'] }, 28 | { namespace: ['logs'] }, 29 | ]) 30 | expect(mockClient.request).toHaveBeenCalledWith({ 31 | method: 'GET', 32 | path: '/v1/namespaces', 33 | query: undefined, 34 | }) 35 | }) 36 | 37 | it('should list namespaces under a parent', async () => { 38 | const mockClient = createMockClient() 39 | vi.mocked(mockClient.request).mockResolvedValue({ 40 | status: 200, 41 | headers: new Headers(), 42 | data: { 43 | namespaces: [ 44 | ['analytics', 'prod'], 45 | ['analytics', 'dev'], 46 | ], 47 | }, 48 | }) 49 | 50 | const ops = new NamespaceOperations(mockClient, '/v1') 51 | const result = await ops.listNamespaces({ namespace: ['analytics'] }) 52 | 53 | expect(result).toEqual([ 54 | { namespace: ['analytics', 'prod'] }, 55 | { namespace: ['analytics', 'dev'] }, 56 | ]) 57 | expect(mockClient.request).toHaveBeenCalledWith({ 58 | method: 'GET', 59 | path: '/v1/namespaces', 60 | query: { parent: 'analytics' }, 61 | }) 62 | }) 63 | 64 | it('should handle multipart namespace parent with separator', async () => { 65 | const mockClient = createMockClient() 66 | vi.mocked(mockClient.request).mockResolvedValue({ 67 | status: 200, 68 | headers: new Headers(), 69 | data: { 70 | namespaces: [['a', 'b', 'c']], 71 | }, 72 | }) 73 | 74 | const ops = new NamespaceOperations(mockClient, '/v1') 75 | await ops.listNamespaces({ namespace: ['a', 'b'] }) 76 | 77 | expect(mockClient.request).toHaveBeenCalledWith({ 78 | method: 'GET', 79 | path: '/v1/namespaces', 80 | query: { parent: 'a\x1Fb' }, 81 | }) 82 | }) 83 | 84 | it('should use prefix when provided', async () => { 85 | const mockClient = createMockClient() 86 | vi.mocked(mockClient.request).mockResolvedValue({ 87 | status: 200, 88 | headers: new Headers(), 89 | data: { namespaces: [] }, 90 | }) 91 | 92 | const ops = new NamespaceOperations(mockClient, '/v1/catalog1') 93 | await ops.listNamespaces() 94 | 95 | expect(mockClient.request).toHaveBeenCalledWith({ 96 | method: 'GET', 97 | path: '/v1/catalog1/namespaces', 98 | query: undefined, 99 | }) 100 | }) 101 | }) 102 | 103 | describe('createNamespace', () => { 104 | it('should create a namespace without properties', async () => { 105 | const mockClient = createMockClient() 106 | vi.mocked(mockClient.request).mockResolvedValue({ 107 | status: 200, 108 | headers: new Headers(), 109 | data: { 110 | namespace: ['analytics'], 111 | }, 112 | }) 113 | 114 | const ops = new NamespaceOperations(mockClient, '/v1') 115 | const result = await ops.createNamespace({ namespace: ['analytics'] }) 116 | 117 | expect(result).toEqual({ namespace: ['analytics'] }) 118 | expect(mockClient.request).toHaveBeenCalledWith({ 119 | method: 'POST', 120 | path: '/v1/namespaces', 121 | body: { 122 | namespace: ['analytics'], 123 | properties: undefined, 124 | }, 125 | }) 126 | }) 127 | 128 | it('should create a namespace with properties', async () => { 129 | const mockClient = createMockClient() 130 | vi.mocked(mockClient.request).mockResolvedValue({ 131 | status: 200, 132 | headers: new Headers(), 133 | data: { 134 | namespace: ['analytics'], 135 | properties: { owner: 'team' }, 136 | }, 137 | }) 138 | 139 | const ops = new NamespaceOperations(mockClient, '/v1') 140 | const result = await ops.createNamespace( 141 | { namespace: ['analytics'] }, 142 | { properties: { owner: 'team' } } 143 | ) 144 | 145 | expect(result).toEqual({ 146 | namespace: ['analytics'], 147 | properties: { owner: 'team' }, 148 | }) 149 | expect(mockClient.request).toHaveBeenCalledWith({ 150 | method: 'POST', 151 | path: '/v1/namespaces', 152 | body: { 153 | namespace: ['analytics'], 154 | properties: { owner: 'team' }, 155 | }, 156 | }) 157 | }) 158 | 159 | it('should create multipart namespace', async () => { 160 | const mockClient = createMockClient() 161 | vi.mocked(mockClient.request).mockResolvedValue({ 162 | status: 200, 163 | headers: new Headers(), 164 | data: { namespace: ['analytics', 'prod'] }, 165 | }) 166 | 167 | const ops = new NamespaceOperations(mockClient, '/v1') 168 | await ops.createNamespace({ namespace: ['analytics', 'prod'] }) 169 | 170 | expect(mockClient.request).toHaveBeenCalledWith({ 171 | method: 'POST', 172 | path: '/v1/namespaces', 173 | body: { 174 | namespace: ['analytics', 'prod'], 175 | properties: undefined, 176 | }, 177 | }) 178 | }) 179 | }) 180 | 181 | describe('dropNamespace', () => { 182 | it('should drop a namespace', async () => { 183 | const mockClient = createMockClient() 184 | vi.mocked(mockClient.request).mockResolvedValue({ 185 | status: 204, 186 | headers: new Headers(), 187 | data: undefined, 188 | }) 189 | 190 | const ops = new NamespaceOperations(mockClient, '/v1') 191 | await ops.dropNamespace({ namespace: ['analytics'] }) 192 | 193 | expect(mockClient.request).toHaveBeenCalledWith({ 194 | method: 'DELETE', 195 | path: '/v1/namespaces/analytics', 196 | }) 197 | }) 198 | 199 | it('should drop multipart namespace with separator', async () => { 200 | const mockClient = createMockClient() 201 | vi.mocked(mockClient.request).mockResolvedValue({ 202 | status: 204, 203 | headers: new Headers(), 204 | data: undefined, 205 | }) 206 | 207 | const ops = new NamespaceOperations(mockClient, '/v1') 208 | await ops.dropNamespace({ namespace: ['analytics', 'prod'] }) 209 | 210 | expect(mockClient.request).toHaveBeenCalledWith({ 211 | method: 'DELETE', 212 | path: '/v1/namespaces/analytics\x1Fprod', 213 | }) 214 | }) 215 | }) 216 | 217 | describe('loadNamespaceMetadata', () => { 218 | it('should load namespace metadata', async () => { 219 | const mockClient = createMockClient() 220 | vi.mocked(mockClient.request).mockResolvedValue({ 221 | status: 200, 222 | headers: new Headers(), 223 | data: { 224 | namespace: ['analytics'], 225 | properties: { 226 | owner: 'data-team', 227 | description: 'Analytics namespace', 228 | }, 229 | }, 230 | }) 231 | 232 | const ops = new NamespaceOperations(mockClient, '/v1') 233 | const result = await ops.loadNamespaceMetadata({ namespace: ['analytics'] }) 234 | 235 | expect(result).toEqual({ 236 | properties: { 237 | owner: 'data-team', 238 | description: 'Analytics namespace', 239 | }, 240 | }) 241 | expect(mockClient.request).toHaveBeenCalledWith({ 242 | method: 'GET', 243 | path: '/v1/namespaces/analytics', 244 | }) 245 | }) 246 | 247 | it('should load metadata for multipart namespace', async () => { 248 | const mockClient = createMockClient() 249 | vi.mocked(mockClient.request).mockResolvedValue({ 250 | status: 200, 251 | headers: new Headers(), 252 | data: { 253 | namespace: ['analytics', 'prod'], 254 | properties: {}, 255 | }, 256 | }) 257 | 258 | const ops = new NamespaceOperations(mockClient, '/v1') 259 | await ops.loadNamespaceMetadata({ namespace: ['analytics', 'prod'] }) 260 | 261 | expect(mockClient.request).toHaveBeenCalledWith({ 262 | method: 'GET', 263 | path: '/v1/namespaces/analytics\x1Fprod', 264 | }) 265 | }) 266 | }) 267 | 268 | describe('namespaceExists', () => { 269 | it('should return true when namespace exists', async () => { 270 | const mockClient = createMockClient() 271 | vi.mocked(mockClient.request).mockResolvedValue({ 272 | status: 200, 273 | headers: new Headers(), 274 | data: undefined, 275 | }) 276 | 277 | const ops = new NamespaceOperations(mockClient, '/v1') 278 | const result = await ops.namespaceExists({ namespace: ['analytics'] }) 279 | 280 | expect(result).toBe(true) 281 | expect(mockClient.request).toHaveBeenCalledWith({ 282 | method: 'HEAD', 283 | path: '/v1/namespaces/analytics', 284 | }) 285 | }) 286 | 287 | it('should return false when namespace does not exist', async () => { 288 | const mockClient = createMockClient() 289 | vi.mocked(mockClient.request).mockRejectedValue( 290 | new IcebergError('Not Found', { status: 404 }) 291 | ) 292 | 293 | const ops = new NamespaceOperations(mockClient, '/v1') 294 | const result = await ops.namespaceExists({ namespace: ['analytics'] }) 295 | 296 | expect(result).toBe(false) 297 | }) 298 | 299 | it('should re-throw non-404 errors', async () => { 300 | const mockClient = createMockClient() 301 | const error = new IcebergError('Server Error', { status: 500 }) 302 | vi.mocked(mockClient.request).mockRejectedValue(error) 303 | 304 | const ops = new NamespaceOperations(mockClient, '/v1') 305 | 306 | await expect(ops.namespaceExists({ namespace: ['analytics'] })).rejects.toThrow(error) 307 | }) 308 | }) 309 | 310 | describe('createNamespaceIfNotExists', () => { 311 | it('should create namespace if it does not exist', async () => { 312 | const mockClient = createMockClient() 313 | vi.mocked(mockClient.request).mockResolvedValueOnce({ 314 | status: 200, 315 | headers: new Headers(), 316 | data: { 317 | namespace: ['analytics'], 318 | properties: { owner: 'data-team' }, 319 | }, 320 | }) 321 | 322 | const ops = new NamespaceOperations(mockClient, '/v1') 323 | await ops.createNamespaceIfNotExists( 324 | { namespace: ['analytics'] }, 325 | { properties: { owner: 'data-team' } } 326 | ) 327 | 328 | expect(mockClient.request).toHaveBeenCalledTimes(1) 329 | expect(mockClient.request).toHaveBeenCalledWith({ 330 | method: 'POST', 331 | path: '/v1/namespaces', 332 | body: { 333 | namespace: ['analytics'], 334 | properties: { owner: 'data-team' }, 335 | }, 336 | }) 337 | }) 338 | 339 | it('should do nothing if namespace already exists', async () => { 340 | const mockClient = createMockClient() 341 | vi.mocked(mockClient.request).mockRejectedValueOnce( 342 | new IcebergError('Namespace already exists', { status: 409 }) 343 | ) 344 | 345 | const ops = new NamespaceOperations(mockClient, '/v1') 346 | await ops.createNamespaceIfNotExists( 347 | { namespace: ['analytics'] }, 348 | { properties: { owner: 'data-team' } } 349 | ) 350 | 351 | expect(mockClient.request).toHaveBeenCalledTimes(1) 352 | }) 353 | 354 | it('should re-throw non-409 errors', async () => { 355 | const mockClient = createMockClient() 356 | const error = new IcebergError('Server Error', { status: 500 }) 357 | vi.mocked(mockClient.request).mockRejectedValue(error) 358 | 359 | const ops = new NamespaceOperations(mockClient, '/v1') 360 | 361 | await expect(ops.createNamespaceIfNotExists({ namespace: ['analytics'] })).rejects.toThrow( 362 | error 363 | ) 364 | }) 365 | }) 366 | }) 367 | -------------------------------------------------------------------------------- /src/catalog/IcebergRestCatalog.ts: -------------------------------------------------------------------------------- 1 | import { createFetchClient } from '../http/createFetchClient' 2 | import type { AuthConfig, HttpClient } from '../http/types' 3 | import { NamespaceOperations } from './namespaces' 4 | import { TableOperations } from './tables' 5 | import type { 6 | CreateTableRequest, 7 | CreateNamespaceResponse, 8 | CommitTableResponse, 9 | NamespaceIdentifier, 10 | NamespaceMetadata, 11 | TableIdentifier, 12 | TableMetadata, 13 | UpdateTableRequest, 14 | DropTableRequest, 15 | } from './types' 16 | 17 | /** 18 | * Access delegation mechanisms supported by the Iceberg REST Catalog. 19 | * 20 | * - `vended-credentials`: Server provides temporary credentials for data access 21 | * - `remote-signing`: Server signs requests on behalf of the client 22 | */ 23 | export type AccessDelegation = 'vended-credentials' | 'remote-signing' 24 | 25 | /** 26 | * Configuration options for the Iceberg REST Catalog client. 27 | */ 28 | export interface IcebergRestCatalogOptions { 29 | /** Base URL of the Iceberg REST Catalog API */ 30 | baseUrl: string 31 | /** Optional catalog name prefix for multi-catalog servers */ 32 | catalogName?: string 33 | /** Authentication configuration */ 34 | auth?: AuthConfig 35 | /** Custom fetch implementation (defaults to globalThis.fetch) */ 36 | fetch?: typeof fetch 37 | /** 38 | * Access delegation mechanisms to request from the server. 39 | * When specified, the X-Iceberg-Access-Delegation header will be sent 40 | * with supported operations (createTable, loadTable). 41 | * 42 | * @example ['vended-credentials'] 43 | * @example ['vended-credentials', 'remote-signing'] 44 | */ 45 | accessDelegation?: AccessDelegation[] 46 | } 47 | 48 | /** 49 | * Client for interacting with an Apache Iceberg REST Catalog. 50 | * 51 | * This class provides methods for managing namespaces and tables in an Iceberg catalog. 52 | * It handles authentication, request formatting, and error handling automatically. 53 | * 54 | * @example 55 | * ```typescript 56 | * const catalog = new IcebergRestCatalog({ 57 | * baseUrl: 'https://my-catalog.example.com/iceberg/v1', 58 | * auth: { type: 'bearer', token: process.env.ICEBERG_TOKEN } 59 | * }); 60 | * 61 | * // Create a namespace 62 | * await catalog.createNamespace({ namespace: ['analytics'] }); 63 | * 64 | * // Create a table 65 | * await catalog.createTable( 66 | * { namespace: ['analytics'] }, 67 | * { 68 | * name: 'events', 69 | * schema: { type: 'struct', fields: [...] } 70 | * } 71 | * ); 72 | * ``` 73 | */ 74 | export class IcebergRestCatalog { 75 | private readonly client: HttpClient 76 | private readonly namespaceOps: NamespaceOperations 77 | private readonly tableOps: TableOperations 78 | private readonly accessDelegation?: string 79 | 80 | /** 81 | * Creates a new Iceberg REST Catalog client. 82 | * 83 | * @param options - Configuration options for the catalog client 84 | */ 85 | constructor(options: IcebergRestCatalogOptions) { 86 | let prefix = 'v1' 87 | if (options.catalogName) { 88 | prefix += `/${options.catalogName}` 89 | } 90 | 91 | const baseUrl = options.baseUrl.endsWith('/') ? options.baseUrl : `${options.baseUrl}/` 92 | 93 | this.client = createFetchClient({ 94 | baseUrl, 95 | auth: options.auth, 96 | fetchImpl: options.fetch, 97 | }) 98 | 99 | // Format accessDelegation as comma-separated string per spec 100 | this.accessDelegation = options.accessDelegation?.join(',') 101 | 102 | this.namespaceOps = new NamespaceOperations(this.client, prefix) 103 | this.tableOps = new TableOperations(this.client, prefix, this.accessDelegation) 104 | } 105 | 106 | /** 107 | * Lists all namespaces in the catalog. 108 | * 109 | * @param parent - Optional parent namespace to list children under 110 | * @returns Array of namespace identifiers 111 | * 112 | * @example 113 | * ```typescript 114 | * // List all top-level namespaces 115 | * const namespaces = await catalog.listNamespaces(); 116 | * 117 | * // List namespaces under a parent 118 | * const children = await catalog.listNamespaces({ namespace: ['analytics'] }); 119 | * ``` 120 | */ 121 | async listNamespaces(parent?: NamespaceIdentifier): Promise { 122 | return this.namespaceOps.listNamespaces(parent) 123 | } 124 | 125 | /** 126 | * Creates a new namespace in the catalog. 127 | * 128 | * @param id - Namespace identifier to create 129 | * @param metadata - Optional metadata properties for the namespace 130 | * @returns Response containing the created namespace and its properties 131 | * 132 | * @example 133 | * ```typescript 134 | * const response = await catalog.createNamespace( 135 | * { namespace: ['analytics'] }, 136 | * { properties: { owner: 'data-team' } } 137 | * ); 138 | * console.log(response.namespace); // ['analytics'] 139 | * console.log(response.properties); // { owner: 'data-team', ... } 140 | * ``` 141 | */ 142 | async createNamespace( 143 | id: NamespaceIdentifier, 144 | metadata?: NamespaceMetadata 145 | ): Promise { 146 | return this.namespaceOps.createNamespace(id, metadata) 147 | } 148 | 149 | /** 150 | * Drops a namespace from the catalog. 151 | * 152 | * The namespace must be empty (contain no tables) before it can be dropped. 153 | * 154 | * @param id - Namespace identifier to drop 155 | * 156 | * @example 157 | * ```typescript 158 | * await catalog.dropNamespace({ namespace: ['analytics'] }); 159 | * ``` 160 | */ 161 | async dropNamespace(id: NamespaceIdentifier): Promise { 162 | await this.namespaceOps.dropNamespace(id) 163 | } 164 | 165 | /** 166 | * Loads metadata for a namespace. 167 | * 168 | * @param id - Namespace identifier to load 169 | * @returns Namespace metadata including properties 170 | * 171 | * @example 172 | * ```typescript 173 | * const metadata = await catalog.loadNamespaceMetadata({ namespace: ['analytics'] }); 174 | * console.log(metadata.properties); 175 | * ``` 176 | */ 177 | async loadNamespaceMetadata(id: NamespaceIdentifier): Promise { 178 | return this.namespaceOps.loadNamespaceMetadata(id) 179 | } 180 | 181 | /** 182 | * Lists all tables in a namespace. 183 | * 184 | * @param namespace - Namespace identifier to list tables from 185 | * @returns Array of table identifiers 186 | * 187 | * @example 188 | * ```typescript 189 | * const tables = await catalog.listTables({ namespace: ['analytics'] }); 190 | * console.log(tables); // [{ namespace: ['analytics'], name: 'events' }, ...] 191 | * ``` 192 | */ 193 | async listTables(namespace: NamespaceIdentifier): Promise { 194 | return this.tableOps.listTables(namespace) 195 | } 196 | 197 | /** 198 | * Creates a new table in the catalog. 199 | * 200 | * @param namespace - Namespace to create the table in 201 | * @param request - Table creation request including name, schema, partition spec, etc. 202 | * @returns Table metadata for the created table 203 | * 204 | * @example 205 | * ```typescript 206 | * const metadata = await catalog.createTable( 207 | * { namespace: ['analytics'] }, 208 | * { 209 | * name: 'events', 210 | * schema: { 211 | * type: 'struct', 212 | * fields: [ 213 | * { id: 1, name: 'id', type: 'long', required: true }, 214 | * { id: 2, name: 'timestamp', type: 'timestamp', required: true } 215 | * ], 216 | * 'schema-id': 0 217 | * }, 218 | * 'partition-spec': { 219 | * 'spec-id': 0, 220 | * fields: [ 221 | * { source_id: 2, field_id: 1000, name: 'ts_day', transform: 'day' } 222 | * ] 223 | * } 224 | * } 225 | * ); 226 | * ``` 227 | */ 228 | async createTable( 229 | namespace: NamespaceIdentifier, 230 | request: CreateTableRequest 231 | ): Promise { 232 | return this.tableOps.createTable(namespace, request) 233 | } 234 | 235 | /** 236 | * Updates an existing table's metadata. 237 | * 238 | * Can update the schema, partition spec, or properties of a table. 239 | * 240 | * @param id - Table identifier to update 241 | * @param request - Update request with fields to modify 242 | * @returns Response containing the metadata location and updated table metadata 243 | * 244 | * @example 245 | * ```typescript 246 | * const response = await catalog.updateTable( 247 | * { namespace: ['analytics'], name: 'events' }, 248 | * { 249 | * properties: { 'read.split.target-size': '134217728' } 250 | * } 251 | * ); 252 | * console.log(response['metadata-location']); // s3://... 253 | * console.log(response.metadata); // TableMetadata object 254 | * ``` 255 | */ 256 | async updateTable( 257 | id: TableIdentifier, 258 | request: UpdateTableRequest 259 | ): Promise { 260 | return this.tableOps.updateTable(id, request) 261 | } 262 | 263 | /** 264 | * Drops a table from the catalog. 265 | * 266 | * @param id - Table identifier to drop 267 | * 268 | * @example 269 | * ```typescript 270 | * await catalog.dropTable({ namespace: ['analytics'], name: 'events' }); 271 | * ``` 272 | */ 273 | async dropTable(id: TableIdentifier, options?: DropTableRequest): Promise { 274 | await this.tableOps.dropTable(id, options) 275 | } 276 | 277 | /** 278 | * Loads metadata for a table. 279 | * 280 | * @param id - Table identifier to load 281 | * @returns Table metadata including schema, partition spec, location, etc. 282 | * 283 | * @example 284 | * ```typescript 285 | * const metadata = await catalog.loadTable({ namespace: ['analytics'], name: 'events' }); 286 | * console.log(metadata.schema); 287 | * console.log(metadata.location); 288 | * ``` 289 | */ 290 | async loadTable(id: TableIdentifier): Promise { 291 | return this.tableOps.loadTable(id) 292 | } 293 | 294 | /** 295 | * Checks if a namespace exists in the catalog. 296 | * 297 | * @param id - Namespace identifier to check 298 | * @returns True if the namespace exists, false otherwise 299 | * 300 | * @example 301 | * ```typescript 302 | * const exists = await catalog.namespaceExists({ namespace: ['analytics'] }); 303 | * console.log(exists); // true or false 304 | * ``` 305 | */ 306 | async namespaceExists(id: NamespaceIdentifier): Promise { 307 | return this.namespaceOps.namespaceExists(id) 308 | } 309 | 310 | /** 311 | * Checks if a table exists in the catalog. 312 | * 313 | * @param id - Table identifier to check 314 | * @returns True if the table exists, false otherwise 315 | * 316 | * @example 317 | * ```typescript 318 | * const exists = await catalog.tableExists({ namespace: ['analytics'], name: 'events' }); 319 | * console.log(exists); // true or false 320 | * ``` 321 | */ 322 | async tableExists(id: TableIdentifier): Promise { 323 | return this.tableOps.tableExists(id) 324 | } 325 | 326 | /** 327 | * Creates a namespace if it does not exist. 328 | * 329 | * If the namespace already exists, returns void. If created, returns the response. 330 | * 331 | * @param id - Namespace identifier to create 332 | * @param metadata - Optional metadata properties for the namespace 333 | * @returns Response containing the created namespace and its properties, or void if it already exists 334 | * 335 | * @example 336 | * ```typescript 337 | * const response = await catalog.createNamespaceIfNotExists( 338 | * { namespace: ['analytics'] }, 339 | * { properties: { owner: 'data-team' } } 340 | * ); 341 | * if (response) { 342 | * console.log('Created:', response.namespace); 343 | * } else { 344 | * console.log('Already exists'); 345 | * } 346 | * ``` 347 | */ 348 | async createNamespaceIfNotExists( 349 | id: NamespaceIdentifier, 350 | metadata?: NamespaceMetadata 351 | ): Promise { 352 | return this.namespaceOps.createNamespaceIfNotExists(id, metadata) 353 | } 354 | 355 | /** 356 | * Creates a table if it does not exist. 357 | * 358 | * If the table already exists, returns its metadata instead. 359 | * 360 | * @param namespace - Namespace to create the table in 361 | * @param request - Table creation request including name, schema, partition spec, etc. 362 | * @returns Table metadata for the created or existing table 363 | * 364 | * @example 365 | * ```typescript 366 | * const metadata = await catalog.createTableIfNotExists( 367 | * { namespace: ['analytics'] }, 368 | * { 369 | * name: 'events', 370 | * schema: { 371 | * type: 'struct', 372 | * fields: [ 373 | * { id: 1, name: 'id', type: 'long', required: true }, 374 | * { id: 2, name: 'timestamp', type: 'timestamp', required: true } 375 | * ], 376 | * 'schema-id': 0 377 | * } 378 | * } 379 | * ); 380 | * ``` 381 | */ 382 | async createTableIfNotExists( 383 | namespace: NamespaceIdentifier, 384 | request: CreateTableRequest 385 | ): Promise { 386 | return this.tableOps.createTableIfNotExists(namespace, request) 387 | } 388 | } 389 | -------------------------------------------------------------------------------- /test/integration/test-local-catalog.integration.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, beforeAll, afterAll } from 'vitest' 2 | import { 3 | IcebergRestCatalog, 4 | IcebergError, 5 | getCurrentSchema, 6 | typesEqual, 7 | isDecimalType, 8 | isFixedType, 9 | } from '../../src/index' 10 | 11 | describe('Local Iceberg REST Catalog Integration', () => { 12 | const catalog = new IcebergRestCatalog({ 13 | baseUrl: 'http://localhost:8181', 14 | auth: { type: 'none' }, 15 | }) 16 | 17 | beforeAll(async () => { 18 | // Cleanup: Drop test namespace if it exists (to make test idempotent) 19 | try { 20 | await catalog.dropTable({ namespace: ['test'], name: 'users' }) 21 | } catch (error) { 22 | if (!(error instanceof IcebergError && error.status === 404)) { 23 | throw error 24 | } 25 | } 26 | 27 | try { 28 | await catalog.dropTable({ namespace: ['test'], name: 'complex_types' }) 29 | } catch (error) { 30 | if (!(error instanceof IcebergError && error.status === 404)) { 31 | throw error 32 | } 33 | } 34 | 35 | try { 36 | await catalog.dropNamespace({ namespace: ['test'] }) 37 | } catch (error) { 38 | if (!(error instanceof IcebergError && error.status === 404)) { 39 | throw error 40 | } 41 | } 42 | }) 43 | 44 | afterAll(async () => { 45 | // Cleanup after all tests 46 | try { 47 | await catalog.dropTable({ namespace: ['test'], name: 'users' }) 48 | } catch { 49 | // ignore 50 | } 51 | try { 52 | await catalog.dropTable({ namespace: ['test'], name: 'complex_types' }) 53 | } catch { 54 | // ignore 55 | } 56 | try { 57 | await catalog.dropNamespace({ namespace: ['test'] }) 58 | } catch { 59 | // ignore 60 | } 61 | }) 62 | 63 | describe('Namespace Operations', () => { 64 | it('should list namespaces', async () => { 65 | const namespaces = await catalog.listNamespaces() 66 | expect(Array.isArray(namespaces)).toBe(true) 67 | }) 68 | 69 | it('should create a namespace', async () => { 70 | await catalog.createNamespace( 71 | { namespace: ['test'] }, 72 | { properties: { owner: 'iceberg-js-test' } } 73 | ) 74 | 75 | const namespaces = await catalog.listNamespaces() 76 | const testNs = namespaces.find((ns) => ns.namespace[0] === 'test') 77 | expect(testNs).toBeDefined() 78 | }) 79 | }) 80 | 81 | describe('Table Operations', () => { 82 | it('should create a table', async () => { 83 | const tableMetadata = await catalog.createTable( 84 | { namespace: ['test'] }, 85 | { 86 | name: 'users', 87 | schema: { 88 | type: 'struct', 89 | fields: [ 90 | { id: 1, name: 'id', type: 'long', required: true }, 91 | { id: 2, name: 'name', type: 'string', required: true }, 92 | { id: 3, name: 'email', type: 'string', required: false }, 93 | ], 94 | 'schema-id': 0, 95 | 'identifier-field-ids': [1], 96 | }, 97 | 'partition-spec': { 98 | 'spec-id': 0, 99 | fields: [], 100 | }, 101 | 'write-order': { 102 | 'order-id': 0, 103 | fields: [], 104 | }, 105 | properties: { 106 | 'write.format.default': 'parquet', 107 | }, 108 | } 109 | ) 110 | 111 | expect(tableMetadata.location).toBeDefined() 112 | expect(tableMetadata.location).toContain('users') 113 | }) 114 | 115 | it('should list tables in namespace', async () => { 116 | const tables = await catalog.listTables({ namespace: ['test'] }) 117 | 118 | expect(Array.isArray(tables)).toBe(true) 119 | const usersTable = tables.find((t) => t.name === 'users') 120 | expect(usersTable).toBeDefined() 121 | }) 122 | 123 | it('should load table metadata', async () => { 124 | const loadedTable = await catalog.loadTable({ namespace: ['test'], name: 'users' }) 125 | 126 | expect(loadedTable.schemas).toBeDefined() 127 | expect(loadedTable['current-schema-id']).toBeDefined() 128 | 129 | const currentSchema = getCurrentSchema(loadedTable) 130 | expect(currentSchema).toBeDefined() 131 | expect(currentSchema?.fields).toHaveLength(3) 132 | expect(currentSchema?.fields[0].name).toBe('id') 133 | }) 134 | 135 | it('should update table properties', async () => { 136 | const result = await catalog.updateTable( 137 | { namespace: ['test'], name: 'users' }, 138 | { 139 | properties: { 140 | 'read.split.target-size': '134217728', 141 | 'write.parquet.compression-codec': 'snappy', 142 | }, 143 | } 144 | ) 145 | 146 | expect(result.metadata).toBeDefined() 147 | }) 148 | }) 149 | 150 | describe('Complex Types', () => { 151 | it('should create table with decimal, fixed, list, map, and struct types', async () => { 152 | const tableMetadata = await catalog.createTable( 153 | { namespace: ['test'] }, 154 | { 155 | name: 'complex_types', 156 | schema: { 157 | type: 'struct', 158 | fields: [ 159 | { id: 1, name: 'id', type: 'long', required: true }, 160 | // Decimal type - string format 161 | { id: 2, name: 'price', type: 'decimal(10,2)', required: false }, 162 | // Fixed type - string format 163 | { id: 3, name: 'hash', type: 'fixed[16]', required: false }, 164 | // List type 165 | { 166 | id: 4, 167 | name: 'tags', 168 | type: { 169 | type: 'list', 170 | 'element-id': 5, 171 | element: 'string', 172 | 'element-required': false, 173 | }, 174 | required: false, 175 | }, 176 | // Map type 177 | { 178 | id: 6, 179 | name: 'metadata', 180 | type: { 181 | type: 'map', 182 | 'key-id': 7, 183 | key: 'string', 184 | 'value-id': 8, 185 | value: 'string', 186 | 'value-required': false, 187 | }, 188 | required: false, 189 | }, 190 | // Nested struct type 191 | { 192 | id: 9, 193 | name: 'address', 194 | type: { 195 | type: 'struct', 196 | fields: [ 197 | { id: 10, name: 'street', type: 'string', required: false }, 198 | { id: 11, name: 'city', type: 'string', required: false }, 199 | { id: 12, name: 'zip', type: 'string', required: false }, 200 | ], 201 | }, 202 | required: false, 203 | }, 204 | // List of structs 205 | { 206 | id: 13, 207 | name: 'contacts', 208 | type: { 209 | type: 'list', 210 | 'element-id': 14, 211 | element: { 212 | type: 'struct', 213 | fields: [ 214 | { id: 15, name: 'name', type: 'string', required: true }, 215 | { id: 16, name: 'phone', type: 'string', required: false }, 216 | ], 217 | }, 218 | 'element-required': false, 219 | }, 220 | required: false, 221 | }, 222 | ], 223 | 'schema-id': 0, 224 | 'identifier-field-ids': [1], 225 | }, 226 | 'partition-spec': { 227 | 'spec-id': 0, 228 | fields: [], 229 | }, 230 | 'write-order': { 231 | 'order-id': 0, 232 | fields: [], 233 | }, 234 | properties: {}, 235 | } 236 | ) 237 | 238 | expect(tableMetadata.location).toBeDefined() 239 | }) 240 | 241 | it('should load complex types table and verify schema', async () => { 242 | const loadedTable = await catalog.loadTable({ 243 | namespace: ['test'], 244 | name: 'complex_types', 245 | }) 246 | 247 | const schema = getCurrentSchema(loadedTable) 248 | expect(schema).toBeDefined() 249 | 250 | const fields = schema!.fields 251 | 252 | // Use typesEqual for decimal/fixed to handle whitespace differences 253 | // Catalog may return "decimal(10, 2)" with space even if we sent "decimal(10,2)" 254 | const priceType = fields.find((f) => f.name === 'price')?.type as string 255 | expect(isDecimalType(priceType)).toBe(true) 256 | expect(typesEqual(priceType, 'decimal(10,2)')).toBe(true) 257 | 258 | const hashType = fields.find((f) => f.name === 'hash')?.type as string 259 | expect(isFixedType(hashType)).toBe(true) 260 | expect(typesEqual(hashType, 'fixed[16]')).toBe(true) 261 | 262 | const tagsField = fields.find((f) => f.name === 'tags') 263 | expect(tagsField?.type).toMatchObject({ type: 'list' }) 264 | 265 | const metadataField = fields.find((f) => f.name === 'metadata') 266 | expect(metadataField?.type).toMatchObject({ type: 'map' }) 267 | 268 | const addressField = fields.find((f) => f.name === 'address') 269 | expect(addressField?.type).toMatchObject({ type: 'struct' }) 270 | }) 271 | }) 272 | 273 | describe('Invalid Types (Error Handling)', () => { 274 | // Helper to attempt creating a table with invalid fields 275 | async function createTableWithFields( 276 | tableName: string, 277 | fields: Array<{ id: number; name: string; type: unknown; required: boolean }> 278 | ) { 279 | return catalog.createTable( 280 | { namespace: ['test'] }, 281 | { 282 | name: tableName, 283 | schema: { 284 | type: 'struct', 285 | fields: fields as never, // cast to bypass type checking for invalid type tests 286 | 'schema-id': 0, 287 | 'identifier-field-ids': [1], 288 | }, 289 | 'partition-spec': { 'spec-id': 0, fields: [] }, 290 | 'write-order': { 'order-id': 0, fields: [] }, 291 | properties: {}, 292 | } 293 | ) 294 | } 295 | 296 | it('should reject invalid primitive type', async () => { 297 | await expect( 298 | createTableWithFields(`invalid_prim_${Date.now()}`, [ 299 | { id: 1, name: 'id', type: 'long', required: true }, 300 | { id: 2, name: 'bad_field', type: 'invalid_type', required: false }, 301 | ]) 302 | ).rejects.toThrow(IcebergError) 303 | }) 304 | 305 | it('should reject decimal without params', async () => { 306 | await expect( 307 | createTableWithFields(`invalid_dec1_${Date.now()}`, [ 308 | { id: 1, name: 'id', type: 'long', required: true }, 309 | { id: 2, name: 'bad_decimal', type: 'decimal', required: false }, 310 | ]) 311 | ).rejects.toThrow(IcebergError) 312 | }) 313 | 314 | it('should reject decimal with empty params', async () => { 315 | await expect( 316 | createTableWithFields(`invalid_dec2_${Date.now()}`, [ 317 | { id: 1, name: 'id', type: 'long', required: true }, 318 | { id: 2, name: 'bad_decimal', type: 'decimal()', required: false }, 319 | ]) 320 | ).rejects.toThrow(IcebergError) 321 | }) 322 | 323 | it('should reject decimal missing scale', async () => { 324 | await expect( 325 | createTableWithFields(`invalid_dec3_${Date.now()}`, [ 326 | { id: 1, name: 'id', type: 'long', required: true }, 327 | { id: 2, name: 'bad_decimal', type: 'decimal(10)', required: false }, 328 | ]) 329 | ).rejects.toThrow(IcebergError) 330 | }) 331 | 332 | it('should reject decimal with non-numeric params', async () => { 333 | await expect( 334 | createTableWithFields(`invalid_dec4_${Date.now()}`, [ 335 | { id: 1, name: 'id', type: 'long', required: true }, 336 | { id: 2, name: 'bad_decimal', type: 'decimal(a,b)', required: false }, 337 | ]) 338 | ).rejects.toThrow(IcebergError) 339 | }) 340 | 341 | it('should reject fixed without length', async () => { 342 | await expect( 343 | createTableWithFields(`invalid_fix1_${Date.now()}`, [ 344 | { id: 1, name: 'id', type: 'long', required: true }, 345 | { id: 2, name: 'bad_fixed', type: 'fixed', required: false }, 346 | ]) 347 | ).rejects.toThrow(IcebergError) 348 | }) 349 | 350 | it('should reject fixed with empty length', async () => { 351 | await expect( 352 | createTableWithFields(`invalid_fix2_${Date.now()}`, [ 353 | { id: 1, name: 'id', type: 'long', required: true }, 354 | { id: 2, name: 'bad_fixed', type: 'fixed[]', required: false }, 355 | ]) 356 | ).rejects.toThrow(IcebergError) 357 | }) 358 | 359 | it('should reject fixed with non-numeric length', async () => { 360 | await expect( 361 | createTableWithFields(`invalid_fix3_${Date.now()}`, [ 362 | { id: 1, name: 'id', type: 'long', required: true }, 363 | { id: 2, name: 'bad_fixed', type: 'fixed[abc]', required: false }, 364 | ]) 365 | ).rejects.toThrow(IcebergError) 366 | }) 367 | 368 | it('should reject list missing element-id', async () => { 369 | await expect( 370 | createTableWithFields(`invalid_list_${Date.now()}`, [ 371 | { id: 1, name: 'id', type: 'long', required: true }, 372 | { 373 | id: 2, 374 | name: 'bad_list', 375 | type: { 376 | type: 'list', 377 | element: 'string', 378 | 'element-required': false, 379 | }, 380 | required: false, 381 | }, 382 | ]) 383 | ).rejects.toThrow(IcebergError) 384 | }) 385 | 386 | it('should reject map missing key-id', async () => { 387 | await expect( 388 | createTableWithFields(`invalid_map1_${Date.now()}`, [ 389 | { id: 1, name: 'id', type: 'long', required: true }, 390 | { 391 | id: 2, 392 | name: 'bad_map', 393 | type: { 394 | type: 'map', 395 | key: 'string', 396 | 'value-id': 3, 397 | value: 'string', 398 | 'value-required': false, 399 | }, 400 | required: false, 401 | }, 402 | ]) 403 | ).rejects.toThrow(IcebergError) 404 | }) 405 | 406 | it('should reject map missing value-id', async () => { 407 | await expect( 408 | createTableWithFields(`invalid_map2_${Date.now()}`, [ 409 | { id: 1, name: 'id', type: 'long', required: true }, 410 | { 411 | id: 2, 412 | name: 'bad_map', 413 | type: { 414 | type: 'map', 415 | 'key-id': 3, 416 | key: 'string', 417 | value: 'string', 418 | 'value-required': false, 419 | }, 420 | required: false, 421 | }, 422 | ]) 423 | ).rejects.toThrow(IcebergError) 424 | }) 425 | 426 | it('should reject object type instead of string', async () => { 427 | // It must be a string: 'decimal(10,2)' 428 | // This test verifies the catalog rejects object format 429 | await expect( 430 | createTableWithFields(`invalid_obj_${Date.now()}`, [ 431 | { id: 1, name: 'id', type: 'long', required: true }, 432 | { 433 | id: 2, 434 | name: 'bad_decimal', 435 | type: { type: 'decimal', precision: 10, scale: 2 }, // object instead of string 436 | required: false, 437 | }, 438 | ]) 439 | ).rejects.toThrow(IcebergError) 440 | }) 441 | }) 442 | }) 443 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # iceberg-js 2 | 3 | [![GitHub](https://img.shields.io/badge/GitHub-iceberg--js-181717?logo=github)](https://github.com/supabase/iceberg-js) 4 | [![Docs](https://img.shields.io/badge/docs-API%20Reference-blue?logo=readthedocs)](https://supabase.github.io/iceberg-js/) 5 | [![License](https://img.shields.io/npm/l/nx.svg?style=flat-square)](./LICENSE) 6 | [![CI](https://github.com/supabase/iceberg-js/actions/workflows/ci.yml/badge.svg)](https://github.com/supabase/iceberg-js/actions/workflows/ci.yml) 7 | [![npm version](https://badge.fury.io/js/iceberg-js.svg)](https://www.npmjs.com/package/iceberg-js) 8 | [![pkg.pr.new](https://pkg.pr.new/badge/supabase/iceberg-js)](https://pkg.pr.new/~/supabase/iceberg-js) 9 | 10 | A small, framework-agnostic JavaScript/TypeScript client for the **Apache Iceberg REST Catalog**. 11 | 12 | ## Motivation 13 | 14 | This library provides JavaScript and TypeScript developers with a straightforward way to interact with Apache Iceberg REST Catalogs. It's designed as a thin HTTP wrapper that mirrors the official REST API, making it easy to manage namespaces and tables from any JS/TS environment. 15 | 16 | ## Goals 17 | 18 | - **REST API wrapper**: Provide a 1:1 mapping to the Iceberg REST Catalog API 19 | - **Type safety**: Full TypeScript support with strongly-typed request/response models 20 | - **Minimal footprint**: No engine-specific logic, no heavy dependencies 21 | - **Stability**: Production-ready for catalog management operations 22 | - **Vendor-agnostic**: Works with any Iceberg REST Catalog implementation 23 | 24 | ## Non-Goals 25 | 26 | This library intentionally does **not** support: 27 | 28 | - **Data operations**: Reading or writing table data (Parquet files, etc.) 29 | - **Query execution**: Use dedicated query engines (Spark, Trino, DuckDB, etc.) 30 | - **Engine integration**: No Spark, Flink, or other engine-specific code 31 | - **Advanced features**: Branching, tagging, time travel queries beyond metadata 32 | - **Views or multi-table transactions** 33 | 34 | These boundaries keep the library focused and maintainable. For data operations, pair this library with a query engine that supports Iceberg. 35 | 36 | ## Features 37 | 38 | - **Generic**: Works with any Iceberg REST Catalog implementation, not tied to any specific vendor 39 | - **Minimal**: Thin HTTP wrapper over the official REST API, no engine-specific logic 40 | - **Type-safe**: First-class TypeScript support with strongly-typed request/response models 41 | - **Fetch-based**: Uses native `fetch` API with support for custom implementations 42 | - **Universal**: Targets Node 20+ and modern browsers (ES2020) 43 | - **Catalog-only**: Focused on catalog operations (no data reading/Parquet support in v0.1.0) 44 | 45 | ## Documentation 46 | 47 | 📚 **Full API documentation**: [supabase.github.io/iceberg-js](https://supabase.github.io/iceberg-js/) 48 | 49 | ## Installation 50 | 51 | ```bash 52 | npm install iceberg-js 53 | ``` 54 | 55 | ## Quick Start 56 | 57 | ```typescript 58 | import { IcebergRestCatalog } from 'iceberg-js' 59 | 60 | const catalog = new IcebergRestCatalog({ 61 | baseUrl: 'https://my-catalog.example.com/iceberg/v1', 62 | auth: { 63 | type: 'bearer', 64 | token: process.env.ICEBERG_TOKEN, 65 | }, 66 | }) 67 | 68 | // Create a namespace 69 | await catalog.createNamespace({ namespace: ['analytics'] }) 70 | 71 | // Create a table 72 | await catalog.createTable( 73 | { namespace: ['analytics'] }, 74 | { 75 | name: 'events', 76 | schema: { 77 | type: 'struct', 78 | fields: [ 79 | { id: 1, name: 'id', type: 'long', required: true }, 80 | { id: 2, name: 'timestamp', type: 'timestamp', required: true }, 81 | { id: 3, name: 'user_id', type: 'string', required: false }, 82 | ], 83 | 'schema-id': 0, 84 | 'identifier-field-ids': [1], 85 | }, 86 | 'partition-spec': { 87 | 'spec-id': 0, 88 | fields: [], 89 | }, 90 | 'write-order': { 91 | 'order-id': 0, 92 | fields: [], 93 | }, 94 | properties: { 95 | 'write.format.default': 'parquet', 96 | }, 97 | } 98 | ) 99 | ``` 100 | 101 | ## API Reference 102 | 103 | ### Constructor 104 | 105 | #### `new IcebergRestCatalog(options)` 106 | 107 | Creates a new catalog client instance. 108 | 109 | **Options:** 110 | 111 | - `baseUrl` (string, required): Base URL of the REST catalog 112 | - `auth` (AuthConfig, optional): Authentication configuration 113 | - `catalogName` (string, optional): Catalog name for multi-catalog servers. When specified, requests are sent to `{baseUrl}/v1/{catalogName}/...`. For example, with `baseUrl: 'https://host.com'` and `catalogName: 'prod'`, requests go to `https://host.com/v1/prod/namespaces` 114 | - `fetch` (typeof fetch, optional): Custom fetch implementation 115 | - `accessDelegation` (AccessDelegation[], optional): Access delegation mechanisms to request from the server 116 | 117 | **Authentication types:** 118 | 119 | ```typescript 120 | // No authentication 121 | { type: 'none' } 122 | 123 | // Bearer token 124 | { type: 'bearer', token: 'your-token' } 125 | 126 | // Custom header 127 | { type: 'header', name: 'X-Custom-Auth', value: 'secret' } 128 | 129 | // Custom function 130 | { type: 'custom', getHeaders: async () => ({ 'Authorization': 'Bearer ...' }) } 131 | ``` 132 | 133 | **Access Delegation:** 134 | 135 | Access delegation allows the catalog server to provide temporary credentials or sign requests on your behalf: 136 | 137 | ```typescript 138 | import { IcebergRestCatalog } from 'iceberg-js' 139 | 140 | const catalog = new IcebergRestCatalog({ 141 | baseUrl: 'https://catalog.example.com/iceberg/v1', 142 | auth: { type: 'bearer', token: 'your-token' }, 143 | // Request vended credentials for data access 144 | accessDelegation: ['vended-credentials'], 145 | }) 146 | 147 | // The server may return temporary credentials in the table metadata 148 | const metadata = await catalog.loadTable({ namespace: ['analytics'], name: 'events' }) 149 | // Use credentials from metadata.config to access table data files 150 | ``` 151 | 152 | Supported delegation mechanisms: 153 | 154 | - `vended-credentials`: Server provides temporary credentials (e.g., AWS STS tokens) for accessing table data 155 | - `remote-signing`: Server signs data access requests on behalf of the client 156 | 157 | ### Namespace Operations 158 | 159 | #### `listNamespaces(parent?: NamespaceIdentifier): Promise` 160 | 161 | List all namespaces, optionally under a parent namespace. 162 | 163 | ```typescript 164 | const namespaces = await catalog.listNamespaces() 165 | // [{ namespace: ['default'] }, { namespace: ['analytics'] }] 166 | 167 | const children = await catalog.listNamespaces({ namespace: ['analytics'] }) 168 | // [{ namespace: ['analytics', 'prod'] }] 169 | ``` 170 | 171 | #### `createNamespace(id: NamespaceIdentifier, metadata?: NamespaceMetadata): Promise` 172 | 173 | Create a new namespace with optional properties. 174 | 175 | ```typescript 176 | await catalog.createNamespace({ namespace: ['analytics'] }, { properties: { owner: 'data-team' } }) 177 | ``` 178 | 179 | #### `dropNamespace(id: NamespaceIdentifier): Promise` 180 | 181 | Drop a namespace. The namespace must be empty. 182 | 183 | ```typescript 184 | await catalog.dropNamespace({ namespace: ['analytics'] }) 185 | ``` 186 | 187 | #### `loadNamespaceMetadata(id: NamespaceIdentifier): Promise` 188 | 189 | Load namespace metadata and properties. 190 | 191 | ```typescript 192 | const metadata = await catalog.loadNamespaceMetadata({ namespace: ['analytics'] }) 193 | // { properties: { owner: 'data-team', ... } } 194 | ``` 195 | 196 | ### Table Operations 197 | 198 | #### `listTables(namespace: NamespaceIdentifier): Promise` 199 | 200 | List all tables in a namespace. 201 | 202 | ```typescript 203 | const tables = await catalog.listTables({ namespace: ['analytics'] }) 204 | // [{ namespace: ['analytics'], name: 'events' }] 205 | ``` 206 | 207 | #### `createTable(namespace: NamespaceIdentifier, request: CreateTableRequest): Promise` 208 | 209 | Create a new table. 210 | 211 | ```typescript 212 | const metadata = await catalog.createTable( 213 | { namespace: ['analytics'] }, 214 | { 215 | name: 'events', 216 | schema: { 217 | type: 'struct', 218 | fields: [ 219 | { id: 1, name: 'id', type: 'long', required: true }, 220 | { id: 2, name: 'timestamp', type: 'timestamp', required: true }, 221 | ], 222 | 'schema-id': 0, 223 | }, 224 | 'partition-spec': { 225 | 'spec-id': 0, 226 | fields: [ 227 | { 228 | source_id: 2, 229 | field_id: 1000, 230 | name: 'ts_day', 231 | transform: 'day', 232 | }, 233 | ], 234 | }, 235 | } 236 | ) 237 | ``` 238 | 239 | #### `loadTable(id: TableIdentifier): Promise` 240 | 241 | Load table metadata. 242 | 243 | ```typescript 244 | const metadata = await catalog.loadTable({ 245 | namespace: ['analytics'], 246 | name: 'events', 247 | }) 248 | ``` 249 | 250 | #### `updateTable(id: TableIdentifier, request: UpdateTableRequest): Promise` 251 | 252 | Update table metadata (schema, partition spec, or properties). 253 | 254 | ```typescript 255 | const updated = await catalog.updateTable( 256 | { namespace: ['analytics'], name: 'events' }, 257 | { 258 | properties: { 'read.split.target-size': '134217728' }, 259 | } 260 | ) 261 | ``` 262 | 263 | #### `dropTable(id: TableIdentifier): Promise` 264 | 265 | Drop a table from the catalog. 266 | 267 | ```typescript 268 | await catalog.dropTable({ namespace: ['analytics'], name: 'events' }) 269 | ``` 270 | 271 | ## Error Handling 272 | 273 | All API errors throw an `IcebergError` with details from the server: 274 | 275 | ```typescript 276 | import { IcebergError } from 'iceberg-js' 277 | 278 | try { 279 | await catalog.loadTable({ namespace: ['test'], name: 'missing' }) 280 | } catch (error) { 281 | if (error instanceof IcebergError) { 282 | console.log(error.status) // 404 283 | console.log(error.icebergType) // 'NoSuchTableException' 284 | console.log(error.message) // 'Table does not exist' 285 | } 286 | } 287 | ``` 288 | 289 | ## TypeScript Types 290 | 291 | The library exports all relevant types: 292 | 293 | ```typescript 294 | import type { 295 | NamespaceIdentifier, 296 | TableIdentifier, 297 | TableSchema, 298 | TableField, 299 | IcebergType, 300 | PartitionSpec, 301 | SortOrder, 302 | CreateTableRequest, 303 | TableMetadata, 304 | AuthConfig, 305 | AccessDelegation, 306 | } from 'iceberg-js' 307 | ``` 308 | 309 | ## Supported Iceberg Types 310 | 311 | The following Iceberg primitive types are supported: 312 | 313 | - `boolean`, `int`, `long`, `float`, `double` 314 | - `string`, `uuid`, `binary` 315 | - `date`, `time`, `timestamp`, `timestamptz` 316 | - `decimal(precision, scale)`, `fixed(length)` 317 | 318 | ## Compatibility 319 | 320 | This package is built to work in **all** Node.js and JavaScript environments: 321 | 322 | | Environment | Module System | Import Method | Status | 323 | | ------------------- | -------------------- | --------------------------------------- | ------------------ | 324 | | Node.js ESM | `"type": "module"` | `import { ... } from 'iceberg-js'` | Fully supported | 325 | | Node.js CommonJS | Default | `const { ... } = require('iceberg-js')` | Fully supported | 326 | | TypeScript ESM | `module: "ESNext"` | `import { ... } from 'iceberg-js'` | Full type support | 327 | | TypeScript CommonJS | `module: "CommonJS"` | `import { ... } from 'iceberg-js'` | Full type support | 328 | | Bundlers | Any | Webpack, Vite, esbuild, Rollup, etc. | Auto-detected | 329 | | Browsers | ESM | `