├── .gitignore ├── docker ├── .gitignore ├── tests │ ├── .gitignore │ ├── .flake8 │ ├── test-requirements.txt │ ├── pyproject.toml │ ├── run.sh │ ├── README.md │ ├── test.py │ └── helpers.py ├── requirements.txt ├── .ice.yaml ├── config.d │ ├── keeper │ │ └── listeners.xml │ ├── swarm │ │ ├── listeners.xml │ │ └── remote_servers.xml │ └── vector │ │ ├── listeners.xml │ │ └── remote_servers.xml ├── users.d │ ├── default-user.xml │ └── root.xml ├── clean-all.sh ├── iceberg_read.py ├── iceberg_setup.py ├── docker-compose.yml └── README.md ├── kubernetes ├── helm │ ├── .gitignore │ ├── Chart.yaml │ ├── templates │ │ ├── keeper.yaml │ │ ├── vector.yaml │ │ └── swarm.yaml │ └── values.yaml ├── terraform │ ├── .gitignore │ ├── README.md │ └── main.tf ├── ice │ ├── .gitignore │ ├── .ice.yaml │ ├── main.tf │ └── README.md ├── .ice.yaml ├── manifests │ ├── nvme │ │ ├── local-storage-pvc.yaml │ │ ├── eks-nvme-ssd-provisioner.yaml │ │ ├── swarm-hostpath.yaml │ │ ├── swarm-local-storage.yaml │ │ ├── local-storage-eks-nvme-ssd.yaml │ │ └── README.md │ ├── gp3-encrypted-fast-storage-class.yaml │ ├── keeper.yaml │ ├── vector.yaml │ ├── README.md │ └── swarm.yaml └── README.md ├── docs ├── images │ ├── logo_horizontal_blue_black.png │ ├── logo_horizontal_blue_white.png │ └── Antalya-Reference-Architecture-2025-02-17.png ├── concepts.md └── reference.md ├── CONTRIBUTING.md ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | work/ 2 | -------------------------------------------------------------------------------- /docker/.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | data/ 3 | -------------------------------------------------------------------------------- /kubernetes/helm/.gitignore: -------------------------------------------------------------------------------- 1 | charts 2 | -------------------------------------------------------------------------------- /kubernetes/terraform/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform* 2 | terraform* 3 | -------------------------------------------------------------------------------- /docker/tests/.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | test_logs/ 3 | __pycache__/ 4 | -------------------------------------------------------------------------------- /kubernetes/ice/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform* 2 | terraform* 3 | tf.export 4 | -------------------------------------------------------------------------------- /docker/requirements.txt: -------------------------------------------------------------------------------- 1 | pandas==2.3.3 2 | pyarrow==21.0.0 3 | pydantic==2.11.10 4 | pyiceberg-core==0.6.0 5 | pyiceberg==0.10.0 6 | -------------------------------------------------------------------------------- /docs/images/logo_horizontal_blue_black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altinity/antalya-examples/HEAD/docs/images/logo_horizontal_blue_black.png -------------------------------------------------------------------------------- /docs/images/logo_horizontal_blue_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altinity/antalya-examples/HEAD/docs/images/logo_horizontal_blue_white.png -------------------------------------------------------------------------------- /kubernetes/.ice.yaml: -------------------------------------------------------------------------------- 1 | # Endpoint must be a bucket id. 2 | uri: http://localhost:5000 3 | bearerToken: foo 4 | httpCacheDir: var/cache/ice/http 5 | -------------------------------------------------------------------------------- /kubernetes/ice/.ice.yaml: -------------------------------------------------------------------------------- 1 | # Endpoint must be a bucket id. 2 | uri: http://localhost:5000 3 | bearerToken: foo 4 | httpCacheDir: var/cache/ice/http 5 | -------------------------------------------------------------------------------- /docs/images/Antalya-Reference-Architecture-2025-02-17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altinity/antalya-examples/HEAD/docs/images/Antalya-Reference-Architecture-2025-02-17.png -------------------------------------------------------------------------------- /docker/tests/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203, W503, E501 4 | exclude = 5 | .git, 6 | __pycache__, 7 | .venv, 8 | venv, 9 | build, 10 | dist -------------------------------------------------------------------------------- /docker/tests/test-requirements.txt: -------------------------------------------------------------------------------- 1 | # Used in tests. 2 | requests==2.31.0 3 | # Code formatting. 4 | black==25.9.0 5 | flake8==7.3.0 6 | isort==7.0.0 7 | # Also need requirements from ../requirements.txt 8 | -------------------------------------------------------------------------------- /docker/.ice.yaml: -------------------------------------------------------------------------------- 1 | # Ports are remapped from 5000 (ice catalog default) 2 | # and 9000 (minio API default) to avoid collisions. 3 | uri: http://localhost:5000 4 | bearerToken: foo 5 | s3: 6 | endpoint: http://localhost:9002 7 | -------------------------------------------------------------------------------- /kubernetes/manifests/nvme/local-storage-pvc.yaml: -------------------------------------------------------------------------------- 1 | # Test PVC to see if we can claim a local volume. 2 | kind: PersistentVolumeClaim 3 | apiVersion: v1 4 | metadata: 5 | name: example-local-claim 6 | spec: 7 | accessModes: 8 | - ReadWriteOnce 9 | resources: 10 | requests: 11 | storage: 5Gi 12 | storageClassName: nvme-ssd 13 | -------------------------------------------------------------------------------- /docker/config.d/keeper/listeners.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | :: 7 | 0.0.0.0 8 | 1 9 | 10 | -------------------------------------------------------------------------------- /docker/config.d/swarm/listeners.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | :: 7 | 0.0.0.0 8 | 1 9 | 10 | -------------------------------------------------------------------------------- /docker/config.d/vector/listeners.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | :: 7 | 0.0.0.0 8 | 1 9 | 10 | -------------------------------------------------------------------------------- /docker/users.d/default-user.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ::1 8 | 127.0.0.1 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /docker/tests/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 88 3 | target-version = ['py38'] 4 | include = '\.pyi?$' 5 | extend-exclude = ''' 6 | /( 7 | # directories 8 | \.eggs 9 | | \.git 10 | | \.hg 11 | | \.mypy_cache 12 | | \.tox 13 | | \.venv 14 | | build 15 | | dist 16 | )/ 17 | ''' 18 | 19 | [tool.isort] 20 | profile = "black" 21 | multi_line_output = 3 22 | line_length = 88 23 | known_first_party = ["test"] -------------------------------------------------------------------------------- /kubernetes/manifests/gp3-encrypted-fast-storage-class.yaml: -------------------------------------------------------------------------------- 1 | # Storage class for fast GP3 volumes. 2 | apiVersion: storage.k8s.io/v1 3 | kind: StorageClass 4 | metadata: 5 | name: gp3-encrypted-fast 6 | provisioner: ebs.csi.aws.com 7 | parameters: 8 | encrypted: "true" 9 | fsType: ext4 10 | throughput: "1000" 11 | iops: "3000" 12 | type: gp3 13 | reclaimPolicy: Delete 14 | volumeBindingMode: WaitForFirstConsumer 15 | allowVolumeExpansion: true 16 | 17 | -------------------------------------------------------------------------------- /kubernetes/helm/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: clickhouse-antalya 3 | version: 0.1.0 4 | description: A Helm chart for deploying Antalya builds of ClickHouse 5 | type: application 6 | appVersion: "24.12.2.20101" 7 | home: https://github.com/Altinity/antalya-examples 8 | maintainers: 9 | - name: Altinity 10 | url: https://altinity.com 11 | sources: 12 | - https://github.com/Altinity/antalya-examples 13 | dependencies: [] 14 | annotations: 15 | category: Database 16 | licenses: Apache-2.0 -------------------------------------------------------------------------------- /docker/clean-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$1" != "-f" ]; then 3 | echo "This script deletes all live Docker containers and volumes!" 4 | echo "Run $0 -f to approve this level of destruction." 5 | exit 1 6 | fi 7 | echo "Killing live containers" 8 | for c in $(docker ps -q) 9 | do 10 | docker kill $c; 11 | done 12 | echo "Removing dead containers" 13 | for c in $(docker ps -q -a) 14 | do 15 | docker rm $c; 16 | done 17 | echo "Removing volumes" 18 | for v in $(docker volume ls -q) 19 | do 20 | docker volume rm $v; 21 | done 22 | -------------------------------------------------------------------------------- /docker/users.d/root.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 53336a676c64c1396553b2b7c92f38126768827c93b64d9142069c10eda7a721 7 | ::/0 8 | 9 | 10 | default 11 | 12 | 13 | default 14 | 15 | 16 | 1 17 | 18 | 19 | 1 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /docker/iceberg_read.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Python script showing how to create and populate a table in Iceberg. 3 | 4 | # Uncomment to see the module search path. 5 | #import sys 6 | #print(sys.path) 7 | 8 | import pyarrow 9 | import pyiceberg 10 | # Allows us to connect to the catalog. 11 | from pyiceberg.catalog import load_catalog 12 | 13 | print("Connect to the catalog") 14 | catalog = load_catalog( 15 | "rest", 16 | **{ 17 | "uri": "http://localhost:5000/", # REST server URL. 18 | "type": "rest", 19 | "token": "foo", 20 | "s3.endpoint": f"http://localhost:9002", # Minio URI and credentials 21 | "s3.access-key-id": "minio", 22 | "s3.secret-access-key": "minio123", 23 | } 24 | ) 25 | 26 | print("Get iceberg.bids data as a Pandas dataframe and print it") 27 | try: 28 | table = catalog.load_table("iceberg.bids") 29 | df = table.scan().to_pandas() 30 | print(df) 31 | except pyiceberg.exceptions.NoSuchTableError: 32 | print("Table iceberg.bids does not exist") 33 | -------------------------------------------------------------------------------- /kubernetes/manifests/keeper.yaml: -------------------------------------------------------------------------------- 1 | # Keeper ensemble resource. Used for registration of swarm clusters. 2 | apiVersion: "clickhouse-keeper.altinity.com/v1" 3 | kind: "ClickHouseKeeperInstallation" 4 | metadata: 5 | name: "keeper" 6 | spec: 7 | configuration: 8 | clusters: 9 | - name: "example" 10 | layout: 11 | replicasCount: 3 12 | defaults: 13 | templates: 14 | podTemplate: default 15 | dataVolumeClaimTemplate: default 16 | templates: 17 | podTemplates: 18 | - name: default 19 | spec: 20 | containers: 21 | - name: clickhouse-keeper 22 | imagePullPolicy: IfNotPresent 23 | image: "altinity/clickhouse-keeper:25.8.9.20207.altinityantalya" 24 | volumeClaimTemplates: 25 | - name: default 26 | metadata: 27 | name: both-paths 28 | spec: 29 | storageClassName: gp3-encrypted 30 | accessModes: 31 | - ReadWriteOnce 32 | resources: 33 | requests: 34 | storage: 25Gi 35 | -------------------------------------------------------------------------------- /kubernetes/helm/templates/keeper.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "clickhouse-keeper.altinity.com/v1" 2 | kind: "ClickHouseKeeperInstallation" 3 | metadata: 4 | name: "keeper" 5 | spec: 6 | configuration: 7 | clusters: 8 | - name: "example" 9 | layout: 10 | replicasCount: {{ .Values.keeper.replicasCount }} 11 | defaults: 12 | templates: 13 | podTemplate: default 14 | dataVolumeClaimTemplate: default 15 | templates: 16 | podTemplates: 17 | - name: default 18 | spec: 19 | containers: 20 | - name: clickhouse-keeper 21 | imagePullPolicy: {{ .Values.keeper.image.pullPolicy }} 22 | image: "{{ .Values.keeper.image.repository }}:{{ .Values.keeper.image.tag }}" 23 | volumeClaimTemplates: 24 | - name: default 25 | metadata: 26 | name: both-paths 27 | spec: 28 | storageClassName: {{ if .Values.keeper.storage.class }}{{ .Values.keeper.storage.class }}{{ else }}{{ .Values.global.storageClass }}{{ end }} 29 | accessModes: 30 | - ReadWriteOnce 31 | resources: 32 | requests: 33 | storage: {{ .Values.keeper.storage.size }} 34 | -------------------------------------------------------------------------------- /docker/tests/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Exit on any error 4 | set -e 5 | 6 | # Parse command line arguments 7 | USE_EXISTING=false 8 | for arg in "$@"; do 9 | case $arg in 10 | --use-existing) 11 | USE_EXISTING=true 12 | shift 13 | ;; 14 | *) 15 | # Unknown option 16 | echo "Usage: $0 [--use-existing]" 17 | echo " --use-existing Use existing Docker Compose setup instead of starting/stopping services" 18 | exit 1 19 | ;; 20 | esac 21 | done 22 | 23 | echo "Setting up Python virtual environment..." 24 | 25 | # Create virtual environment if it doesn't exist 26 | if [ ! -d "venv" ]; then 27 | python3 -m venv venv 28 | echo "Virtual environment created." 29 | else 30 | echo "Virtual environment already exists." 31 | fi 32 | 33 | # Activate virtual environment 34 | source venv/bin/activate 35 | echo "Virtual environment activated." 36 | 37 | # Install dependencies 38 | echo "Installing dependencies..." 39 | pip install -r ../requirements.txt 40 | pip install -r test-requirements.txt 41 | 42 | # Set environment variable if using existing setup 43 | if $USE_EXISTING; then 44 | echo "Using existing Docker Compose setup..." 45 | export SKIP_DOCKER_SETUP=true 46 | fi 47 | 48 | # Run tests 49 | echo "Running tests..." 50 | python test.py 51 | 52 | echo "Tests completed." 53 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Antalya Examples 2 | 3 | ## Welcome 4 | We gladly accept PRs and issues to improve this project. 5 | 6 | ## Development Guidelines 7 | 8 | 1. Submit a GitHub issue if you want to discuss design prior to creating a pull request. 9 | 2. Format files in accordance with the language 10 | conventions. 11 | 3. Include tests. If there are existing tests extend them with new cases. Otherwise create automated tests. 12 | 4. Keep documentation current. Update README.md files to describe new behavior and remove obsolete instructions. 13 | 5. Keep documentation concise. Provide only essential information and commands needed to complete tasks. 14 | 6. Place Apache 2.0 header comments in Python files. 15 | 16 | ## Testing Conventions 17 | 18 | 1. Use Python unittest to write tests. 19 | 2. Write a docstring on the unit test class to decribe the test. 20 | 3. Write a docstring on each test case method to document the proposition that it tests. 21 | 4. Keep code as short as possible. Put repeated test code in helper functions. 22 | 5. Dump all information needed to debug failures. 23 | 24 | ## Use of AI tools 25 | 26 | Feel free to use AI tools but add this file to the context by telling 27 | your AI agent to follow these rules for design and code gen activities. 28 | 29 | 1. AI agents are required to read and follow these conventions. 30 | 2. Developers are required to verify all code, including code generated 31 | by AI agents. 32 | 33 | -------------------------------------------------------------------------------- /kubernetes/terraform/README.md: -------------------------------------------------------------------------------- 1 | # Installing AWS EKS using terraform 2 | 3 | This directory shows how to use the Altinity 4 | [terraform-aws-eks-clickhouse](https://github.com/Altinity/terraform-aws-eks-clickhouse) 5 | module to stand up an EKS cluster. 6 | 7 | ## Description 8 | 9 | The main.tf sets up an EKS cluster in a single AZ. The cluster has 10 | 3 node groups. 11 | 12 | * clickhouse - VMs for ClickHouse permanent nodes 13 | * clickhouse-swarm - VMs for ClickHouse swarm nodes 14 | * system - VMs for system nodes including ClickHouse Keeper 15 | 16 | You can extend the example to multiple AZs by uncommenting code lines 17 | appropriately. 18 | 19 | ## Installation 20 | 21 | 1. Edit the main.tf file. 22 | 2. Update the cluster name, region, and any other parameters you wish to change. 23 | 3. Follow the installation instructions. Typical commands are shown below. 24 | 25 | ``` 26 | terraform init 27 | terraform apply 28 | aws eks update-kubeconfig --name my-eks-cluster 29 | ``` 30 | 31 | ## Trouble-shooting 32 | 33 | Use aws eks commands to check configuration of the deployed cluster. 34 | Here's how to see available EKS clusters in the region. 35 | 36 | ``` 37 | aws eks list-clusters 38 | ``` 39 | 40 | Show the node groups in a single cluster. 41 | 42 | ``` 43 | aws eks list-nodegroups --cluster=my-eks-cluster 44 | ``` 45 | 46 | Show the details of a node group. Helpful to ensure you have auto-scaling 47 | limits set correctly. 48 | ``` 49 | aws eks describe-nodegroup --cluster=my-eks-cluster \ 50 | --nodegroup=clickhouse-20250224214119166800000016 51 | ``` 52 | -------------------------------------------------------------------------------- /kubernetes/manifests/nvme/eks-nvme-ssd-provisioner.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | namespace: kube-system 5 | name: eks-nvme-ssd-provisioner 6 | labels: 7 | app: eks-nvme-ssd-provisioner 8 | spec: 9 | selector: 10 | matchLabels: 11 | name: eks-nvme-ssd-provisioner 12 | template: 13 | metadata: 14 | labels: 15 | name: eks-nvme-ssd-provisioner 16 | spec: 17 | automountServiceAccountToken: false 18 | nodeSelector: 19 | aws.amazon.com/eks-local-ssd: "true" 20 | tolerations: 21 | # Run on clickhouse nodes that are dedicated to swarm servers. 22 | - key: "antalya" 23 | operator: "Equal" 24 | value: "nvme-swarm" 25 | effect : "NoSchedule" 26 | - key: "dedicated" 27 | operator: "Equal" 28 | value: "clickhouse" 29 | effect : "NoSchedule" 30 | priorityClassName: system-node-critical 31 | containers: 32 | - name: eks-nvme-ssd-provisioner 33 | #image: goldfin/eks-nvme-ssd-provisioner:latest 34 | image: goldfin/eks-nvme-ssd-provisioner:latest 35 | imagePullPolicy: Always 36 | securityContext: 37 | privileged: true 38 | volumeMounts: 39 | - mountPath: /pv-disks 40 | name: pv-disks 41 | mountPropagation: "Bidirectional" 42 | - mountPath: /nvme 43 | name: nvme 44 | mountPropagation: "Bidirectional" 45 | volumes: 46 | - name: pv-disks 47 | hostPath: 48 | path: /pv-disks 49 | - name: nvme 50 | hostPath: 51 | path: /nvme 52 | -------------------------------------------------------------------------------- /kubernetes/ice/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | source = "hashicorp/aws" 5 | version = "~> 6.0" 6 | } 7 | } 8 | } 9 | 10 | provider "aws" {} 11 | 12 | variable "catalog_bucket" { 13 | description = "Name of the existing S3 bucket to use for the Ice catalog (set via TF_VAR_catalog_bucket or CATALOG_BUCKET)" 14 | type = string 15 | } 16 | 17 | locals { 18 | sqs_queue_prefix = "ice-s3watch" 19 | } 20 | 21 | data "aws_s3_bucket" "this" { 22 | bucket = var.catalog_bucket 23 | } 24 | 25 | resource "aws_sqs_queue" "this" { 26 | name_prefix = local.sqs_queue_prefix 27 | } 28 | 29 | resource "aws_sqs_queue_policy" "this" { 30 | queue_url = aws_sqs_queue.this.id 31 | policy = data.aws_iam_policy_document.queue.json 32 | } 33 | 34 | data "aws_iam_policy_document" "queue" { 35 | statement { 36 | effect = "Allow" 37 | 38 | principals { 39 | type = "*" 40 | identifiers = ["*"] 41 | } 42 | 43 | actions = ["sqs:SendMessage"] 44 | resources = [aws_sqs_queue.this.arn] 45 | 46 | condition { 47 | test = "ArnEquals" 48 | variable = "aws:SourceArn" 49 | values = [data.aws_s3_bucket.this.arn] 50 | } 51 | } 52 | } 53 | 54 | resource "aws_s3_bucket_notification" "this" { 55 | bucket = data.aws_s3_bucket.this.id 56 | 57 | queue { 58 | queue_arn = aws_sqs_queue.this.arn 59 | events = ["s3:ObjectCreated:*"] 60 | filter_suffix = ".parquet" 61 | } 62 | } 63 | 64 | output "s3_bucket_name" { 65 | value = data.aws_s3_bucket.this.id 66 | } 67 | 68 | output "sqs_queue_url" { 69 | value = aws_sqs_queue.this.id 70 | } 71 | -------------------------------------------------------------------------------- /docker/config.d/swarm/remote_servers.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 1 4 | 5 | 6 | 16 | 17 | 18 | keeper 19 | 9181 20 | 21 | 22 | 23 | 24 | 25 | 26 | /clickhouse/discovery/swarm 27 | secret_key 28 | 29 | 30 | 31 | 32 | secret_key 33 | 34 | true 35 | 36 | vector 37 | 9000 38 | 39 | 40 | 41 | true 42 | 43 | swarm-1 44 | 9000 45 | 46 | 47 | 48 | true 49 | 50 | swarm-2 51 | 9000 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /docker/tests/README.md: -------------------------------------------------------------------------------- 1 | # Docker Test Framework 2 | 3 | Tests for the Antalya docker compose examples. 4 | 5 | ## Setup 6 | 7 | 1. Install dependencies: 8 | ```bash 9 | pip install -r requirements.txt 10 | pip install -r test-requirements.txt 11 | ``` 12 | 13 | 2. (Optional) Install development tools for code formatting: 14 | ```bash 15 | pip install black isort flake8 16 | ``` 17 | 18 | ## Running Tests 19 | 20 | Run the test suite from the docker/tests directory: 21 | 22 | **Standard mode (starts/stops Docker Compose):** 23 | ```bash 24 | ./run.sh 25 | ``` 26 | 27 | **Use existing Docker Compose setup:** 28 | ```bash 29 | ./run.sh --use-existing 30 | ``` 31 | 32 | **Direct Python execution:** 33 | ```bash 34 | python test.py 35 | ``` 36 | 37 | ## How it works 38 | 39 | - `setUpClass()`: Runs `docker compose up -d` before any tests (unless `--use-existing` is used) 40 | - `tearDownClass()`: Captures container logs and runs `docker compose down` after all tests complete (only if services were started by the framework) 41 | - Container logs are automatically saved to `test_logs/` directory with timestamps for debugging 42 | - The framework includes sample HTTP tests and Python script execution tests 43 | - Tests will pass if the HTTP response matches the expected status code 44 | - The helper function allows specifying a single expected status code for precise validation 45 | 46 | ## Log Files 47 | 48 | Container logs are automatically captured in the `test_logs/` directory: 49 | - Individual service logs: `{service_name}_{timestamp}.log` 50 | - Combined logs: `combined_{timestamp}.log` 51 | - Logs include timestamps and are captured whether tests pass or fail 52 | 53 | ## Code Formatting 54 | 55 | Format code, sort imports, and check style with flake8 56 | 57 | ``` 58 | black . && isort . && flake8 . 59 | ``` 60 | 61 | ## Adding More Tests 62 | 63 | Create additional test classes that inherit from `DockerTestFramework` 64 | to add more test cases. The Docker Compose services will be managed 65 | automatically. 66 | -------------------------------------------------------------------------------- /kubernetes/helm/values.yaml: -------------------------------------------------------------------------------- 1 | # Global settings 2 | global: 3 | storageClass: "gp3-encrypted" 4 | 5 | # ClickHouse Keeper settings 6 | keeper: 7 | enabled: true 8 | replicasCount: 3 9 | storage: 10 | class: "" # Defaults to global.storageClass if empty 11 | size: "25Gi" 12 | image: 13 | repository: "altinity/clickhouse-keeper" 14 | tag: "25.3.3.20186.altinityantalya" 15 | pullPolicy: "IfNotPresent" 16 | 17 | # ClickHouse Vector settings 18 | vector: 19 | enabled: true 20 | shardsCount: 1 21 | replicas: 22 | count: 1 23 | storage: 24 | class: "" # Defaults to global.storageClass if empty 25 | size: "50Gi" 26 | nodeSelector: 27 | "node.kubernetes.io/instance-type": "m6i.large" 28 | tolerations: 29 | - key: "dedicated" 30 | operator: "Equal" 31 | value: "clickhouse" 32 | effect: "NoSchedule" 33 | image: 34 | repository: "altinity/clickhouse-server" 35 | tag: "25.3.3.20186.altinityantalya" 36 | 37 | # ClickHouse Swarm settings 38 | swarm: 39 | enabled: true 40 | shardsCount: 1 41 | replicas: 42 | count: 1 43 | storage: 44 | class: "" # Defaults to global.storageClass if empty 45 | size: "50Gi" 46 | nodeSelector: 47 | "node.kubernetes.io/instance-type": "m6i.xlarge" 48 | tolerations: 49 | - key: "antalya" 50 | operator: "Equal" 51 | value: "swarm" 52 | effect: "NoSchedule" 53 | - key: "dedicated" 54 | operator: "Equal" 55 | value: "clickhouse" 56 | effect: "NoSchedule" 57 | affinity: 58 | podAntiAffinity: 59 | requiredDuringSchedulingIgnoredDuringExecution: 60 | - labelSelector: 61 | matchExpressions: 62 | - key: "clickhouse.altinity.com/app" 63 | operator: In 64 | values: 65 | - "chop" 66 | topologyKey: "kubernetes.io/hostname" 67 | image: 68 | repository: "altinity/clickhouse-server" 69 | tag: "25.3.3.20186.altinityantalya" 70 | -------------------------------------------------------------------------------- /docker/config.d/vector/remote_servers.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 1 4 | 5 | 6 | 16 | 17 | 18 | keeper 19 | 9181 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | /clickhouse/discovery/swarm 28 | secret_key 29 | 30 | true 31 | 32 | 33 | 34 | 35 | secret_key 36 | 37 | true 38 | 39 | vector 40 | 9000 41 | 42 | 43 | 44 | true 45 | 46 | swarm-1 47 | 9000 48 | 49 | 50 | 51 | true 52 | 53 | swarm-2 54 | 9000 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /kubernetes/manifests/vector.yaml: -------------------------------------------------------------------------------- 1 | # Vector cluster used to initiate queries. 2 | apiVersion: "clickhouse.altinity.com/v1" 3 | kind: "ClickHouseInstallation" 4 | metadata: 5 | name: "vector" 6 | spec: 7 | configuration: 8 | clusters: 9 | - name: "example" 10 | layout: 11 | replicas: 12 | - templates: 13 | podTemplate: replica 14 | volumeClaimTemplate: storage 15 | shardsCount: 1 16 | templates: 17 | zookeeper: 18 | nodes: 19 | - host: keeper-keeper 20 | port: 2181 21 | files: 22 | config.d/remote_servers.xml: | 23 | 24 | 1 25 | 26 | 27 | 28 | 29 | /clickhouse/discovery/swarm 30 | secret_key 31 | 32 | true 33 | 34 | 35 | 36 | 37 | templates: 38 | podTemplates: 39 | - name: replica 40 | spec: 41 | serviceAccountName: ice-rest-catalog 42 | nodeSelector: 43 | node.kubernetes.io/instance-type: m6i.large 44 | containers: 45 | - name: clickhouse 46 | image: altinity/clickhouse-server:25.8.9.20207.altinityantalya 47 | tolerations: 48 | - key: "dedicated" 49 | operator: "Equal" 50 | value: "clickhouse" 51 | effect : "NoSchedule" 52 | volumeClaimTemplates: 53 | - name: storage 54 | # Uncomment for prod systems. You will then need to delete PVCs manually. 55 | # reclaimPolicy: Retain 56 | spec: 57 | storageClassName: gp3-encrypted 58 | accessModes: 59 | - ReadWriteOnce 60 | resources: 61 | requests: 62 | storage: 50Gi 63 | -------------------------------------------------------------------------------- /kubernetes/manifests/README.md: -------------------------------------------------------------------------------- 1 | # Installing Vector and Swarm clusters using manifest files 2 | 3 | This directory shows how to set up an Antalya swarm cluster using 4 | Kubernetes manifest files. The following manifests are provided. 5 | 6 | * keeper.yaml - Sets up a 3-node keeper ensemble. 7 | * swarm.yaml - Sets up a 4-node swarm cluster. 8 | * vector.yaml - Sets up a 1-node "vector" cluster. This is where to issue queries. 9 | 10 | IMPORTANT NOTE: The node selectors and tolerations match node pool 11 | default settings from [main.tf](../terraform/main.tf). If you change 12 | those settings you will need to update the manifests according so that 13 | pods can be scheduled. 14 | 15 | ## Prerequisites 16 | 17 | Install the latest production version of the [Altinity Kubernetes Operator 18 | for ClickHouse](https://github.com/Altinity/clickhouse-operator). 19 | 20 | ``` 21 | kubectl apply -f https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/deploy/operator/clickhouse-operator-install-bundle.yaml 22 | ``` 23 | 24 | ## Installation 25 | 26 | Use Kubectl to install manifests in your default namespace. 27 | ``` 28 | kubectl apply -f gp3-encrypted-fast-storage-class.yaml 29 | kubectl apply -f keeper.yaml 30 | kubectl apply -f swarm.yaml 31 | kubectl apply -f vector.yaml 32 | ``` 33 | 34 | The swarm servers use EBS volumes which integrate well with cluster autoscaling 35 | but are suboptimal for caches. 36 | 37 | ## NVMe SSD swarm support (experimental) 38 | 39 | The [nvme](./nvme) directory contains work in progress. See the 40 | [README.md](nvme/README.md) for more information. Otherwise skip this 41 | section. 42 | 43 | ## Verify installation 44 | 45 | Connect to the vector server and confirm that all swarm cluster hosts are visible. 46 | 47 | ``` 48 | kubectl exec -it chi-vector-example-0-0-0 -- clickhouse-client 49 | ... 50 | SELECT cluster, groupArray(host_name) 51 | FROM system.clusters 52 | GROUP BY cluster ORDER BY cluster ASC FORMAT Vertical 53 | ``` 54 | 55 | You should see the swarm cluster in the last line with 4 hosts listed. Confirm that 56 | all hosts are responsive with the following query. 57 | 58 | ``` 59 | SELECT hostName(), version() 60 | FROM clusterAllReplicas('swarm', system.one) 61 | ORDER BY 1 ASC 62 | ``` 63 | 64 | Your setup is now ready for use. 65 | -------------------------------------------------------------------------------- /kubernetes/manifests/nvme/swarm-hostpath.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "clickhouse.altinity.com/v1" 2 | kind: "ClickHouseInstallation" 3 | metadata: 4 | name: "nvme-swarm" 5 | spec: 6 | configuration: 7 | clusters: 8 | - name: "example" 9 | layout: 10 | replicas: 11 | - templates: 12 | podTemplate: replica 13 | shardsCount: 4 14 | templates: 15 | zookeeper: 16 | nodes: 17 | - host: keeper-keeper 18 | port: 2181 19 | files: 20 | config.d/remote_servers.xml: | 21 | 22 | 1 23 | 24 | 25 | 26 | 27 | /clickhouse/discovery/swarm 28 | secret_key 29 | 30 | 31 | 32 | 33 | templates: 34 | podTemplates: 35 | - name: replica 36 | spec: 37 | nodeSelector: 38 | node.kubernetes.io/instance-type: i8g.xlarge 39 | tolerations: 40 | # Run on clickhouse nodes that are dedicated to swarm servers. 41 | - key: "antalya" 42 | operator: "Equal" 43 | value: "nvme-swarm" 44 | effect : "NoSchedule" 45 | - key: "dedicated" 46 | operator: "Equal" 47 | value: "clickhouse" 48 | effect : "NoSchedule" 49 | affinity: 50 | # Specify Pod anti-affinity to Pods with the same label "/app" on the same "hostname" 51 | podAntiAffinity: 52 | requiredDuringSchedulingIgnoredDuringExecution: 53 | - labelSelector: 54 | matchExpressions: 55 | - key: "clickhouse.altinity.com/app" 56 | operator: In 57 | values: 58 | - "chop" 59 | topologyKey: "kubernetes.io/hostname" 60 | volumes: 61 | - name: local-path 62 | hostPath: 63 | path: /nvme/disk/clickhouse 64 | type: DirectoryOrCreate 65 | containers: 66 | - name: clickhouse 67 | image: altinity/clickhouse-server:24.12.2.20203.altinityantalya 68 | volumeMounts: 69 | # Specify reference to volume on local filesystem 70 | - name: local-path 71 | mountPath: /var/lib/clickhouse 72 | -------------------------------------------------------------------------------- /docker/tests/test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import unittest 17 | from urllib.parse import quote 18 | 19 | from helpers import DockerHelper, generate_basic_auth_header, http_get_helper, run_python_script_helper 20 | 21 | 22 | class DockerTestFramework(unittest.TestCase): 23 | """Docker test framework that manages Docker Compose services and captures logs""" 24 | _docker_helper = None 25 | 26 | @classmethod 27 | def setUpClass(cls): 28 | """Start Docker Compose services before running tests or verify existing setup""" 29 | # Set up directory paths 30 | test_dir = os.path.dirname(os.path.abspath(__file__)) 31 | 32 | # Initialize Docker helper and setup services 33 | cls._docker_helper = DockerHelper(test_dir) 34 | cls._docker_helper.setup_services() 35 | 36 | @classmethod 37 | def tearDownClass(cls): 38 | """Capture logs and stop Docker Compose services if we started them""" 39 | if cls._docker_helper: 40 | cls._docker_helper.cleanup_services() 41 | 42 | def test_ice_catalog_liveness(self): 43 | """Confirm ice catalog on 5000 can list namespaces""" 44 | http_get_helper(self, "http://localhost:5000/v1/namespaces", auth_header="Bearer foo") 45 | 46 | def test_vector_server_liveness(self): 47 | """Confirm ClickHouse vector server responds to ping""" 48 | http_get_helper(self, "http://localhost:8123/ping") 49 | 50 | def test_vector_server_version(self): 51 | """Confirm ClickHouse vector server can list version""" 52 | sql = quote("select version()") 53 | basic_auth = generate_basic_auth_header("root", "topsecret") 54 | http_get_helper(self, f"http://localhost:8123?query={sql}", auth_header=basic_auth) 55 | 56 | def test_iceberg_python_scripts(self): 57 | """Confirm that iceberg py scripts run without errors""" 58 | run_python_script_helper(self, "iceberg_setup.py") 59 | run_python_script_helper(self, "iceberg_read.py") 60 | 61 | 62 | if __name__ == "__main__": 63 | unittest.main(verbosity=2) 64 | -------------------------------------------------------------------------------- /kubernetes/helm/templates/vector.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "clickhouse.altinity.com/v1" 2 | kind: "ClickHouseInstallation" 3 | metadata: 4 | name: "vector" 5 | spec: 6 | configuration: 7 | clusters: 8 | - name: "example" 9 | layout: 10 | replicas: 11 | - templates: 12 | podTemplate: replica 13 | volumeClaimTemplate: storage 14 | shardsCount: {{ .Values.vector.shardsCount }} 15 | templates: 16 | zookeeper: 17 | nodes: 18 | - host: keeper-keeper 19 | port: 2181 20 | files: 21 | config.d/remote_servers.xml: | 22 | 23 | 1 24 | 25 | 26 | 27 | 28 | /clickhouse/discovery/swarm 29 | secret_key 30 | 31 | true 32 | 33 | 34 | 35 | 36 | templates: 37 | podTemplates: 38 | - name: replica 39 | spec: 40 | {{ if .Values.vector.replicas.nodeSelector }} 41 | nodeSelector: 42 | {{ toYaml .Values.vector.replicas.nodeSelector | indent 12 }} 43 | {{ end }} 44 | {{ if .Values.vector.replicas.tolerations }} 45 | tolerations: 46 | {{ toYaml .Values.vector.replicas.tolerations | indent 12 }} 47 | {{ end }} 48 | {{ if .Values.vector.replicas.affinity }} 49 | affinity: 50 | {{ toYaml .Values.vector.replicas.affinity | indent 12 }} 51 | {{ end }} 52 | containers: 53 | - name: clickhouse 54 | image: {{ .Values.vector.image.repository }}:{{ .Values.vector.image.tag }} 55 | volumeClaimTemplates: 56 | - name: storage 57 | # Uncomment for prod systems. You will then need to delete PVCs manually. 58 | # reclaimPolicy: Retain 59 | spec: 60 | {{- $storageClass := "" }} 61 | {{- if .Values.vector.replicas.storage.class }} 62 | {{- $storageClass = .Values.vector.replicas.storage.class }} 63 | {{- else }} 64 | {{- $storageClass = .Values.global.storageClass }} 65 | {{- end }} 66 | {{- if $storageClass }} 67 | storageClassName: {{ $storageClass }} 68 | {{- end }} 69 | accessModes: 70 | - ReadWriteOnce 71 | resources: 72 | requests: 73 | storage: {{ .Values.vector.replicas.storage.size }} 74 | -------------------------------------------------------------------------------- /kubernetes/manifests/nvme/swarm-local-storage.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "clickhouse.altinity.com/v1" 2 | kind: "ClickHouseInstallation" 3 | metadata: 4 | name: "nvme-swarm" 5 | spec: 6 | configuration: 7 | clusters: 8 | - name: "example" 9 | layout: 10 | replicas: 11 | - templates: 12 | podTemplate: replica 13 | volumeClaimTemplate: storage 14 | shardsCount: 2 15 | templates: 16 | zookeeper: 17 | nodes: 18 | - host: keeper-keeper 19 | port: 2181 20 | files: 21 | config.d/remote_servers.xml: | 22 | 23 | 1 24 | 25 | 26 | 27 | 28 | /clickhouse/discovery/swarm 29 | secret_key 30 | 31 | 32 | 33 | 34 | templates: 35 | podTemplates: 36 | - name: replica 37 | spec: 38 | nodeSelector: 39 | node.kubernetes.io/instance-type: i8g.xlarge 40 | tolerations: 41 | # Run on clickhouse nodes that are dedicated to swarm servers. 42 | - key: "antalya" 43 | operator: "Equal" 44 | value: "nvme-swarm" 45 | effect : "NoSchedule" 46 | - key: "dedicated" 47 | operator: "Equal" 48 | value: "clickhouse" 49 | effect : "NoSchedule" 50 | affinity: 51 | # Specify Pod anti-affinity to Pods with the same label "/app" on the same "hostname" 52 | podAntiAffinity: 53 | requiredDuringSchedulingIgnoredDuringExecution: 54 | - labelSelector: 55 | matchExpressions: 56 | - key: "clickhouse.altinity.com/app" 57 | operator: In 58 | values: 59 | - "chop" 60 | topologyKey: "kubernetes.io/hostname" 61 | containers: 62 | - name: clickhouse 63 | image: altinity/clickhouse-server:24.12.2.20203.altinityantalya 64 | volumeClaimTemplates: 65 | - name: storage 66 | # Uncomment for prod systems. You will then need to delete PVCs manually. 67 | # reclaimPolicy: Retain 68 | spec: 69 | storageClassName: local-path 70 | accessModes: 71 | - ReadWriteOnce 72 | resources: 73 | requests: 74 | storage: 50Gi 75 | -------------------------------------------------------------------------------- /kubernetes/helm/templates/swarm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "clickhouse.altinity.com/v1" 2 | kind: "ClickHouseInstallation" 3 | metadata: 4 | name: "swarm" 5 | spec: 6 | configuration: 7 | clusters: 8 | - name: "example" 9 | layout: 10 | replicas: 11 | - templates: 12 | podTemplate: replica 13 | volumeClaimTemplate: storage 14 | shardsCount: {{ .Values.swarm.shardsCount }} 15 | templates: 16 | zookeeper: 17 | nodes: 18 | - host: keeper-keeper 19 | port: 2181 20 | files: 21 | config.d/remote_servers.xml: | 22 | 23 | 1 24 | 25 | 26 | 27 | 28 | /clickhouse/discovery/swarm 29 | secret_key 30 | 31 | 32 | 33 | 34 | templates: 35 | podTemplates: 36 | - name: replica 37 | spec: 38 | {{ if .Values.swarm.replicas.nodeSelector }} 39 | nodeSelector: 40 | {{ toYaml .Values.swarm.replicas.nodeSelector | indent 12 }} 41 | {{ end }} 42 | {{ if .Values.swarm.replicas.tolerations }} 43 | tolerations: 44 | {{ toYaml .Values.swarm.replicas.tolerations | indent 12 }} 45 | {{ end }} 46 | {{ if .Values.swarm.replicas.affinity }} 47 | affinity: 48 | # Specify Pod anti-affinity to Pods with the same label "/app" on the same "hostname" 49 | podAntiAffinity: 50 | {{ toYaml .Values.swarm.replicas.affinity.podAntiAffinity | indent 14 }} 51 | {{ end }} 52 | containers: 53 | - name: clickhouse 54 | image: {{ .Values.swarm.image.repository }}:{{ .Values.swarm.image.tag }} 55 | volumeClaimTemplates: 56 | - name: storage 57 | # Uncomment for prod systems. You will then need to delete PVCs manually. 58 | # reclaimPolicy: Retain 59 | spec: 60 | {{- $storageClass := "" }} 61 | {{- if .Values.swarm.replicas.storage.class }} 62 | {{- $storageClass = .Values.swarm.replicas.storage.class }} 63 | {{- else }} 64 | {{- $storageClass = .Values.global.storageClass }} 65 | {{- end }} 66 | {{- if $storageClass }} 67 | storageClassName: {{ $storageClass }} 68 | {{- end }} 69 | accessModes: 70 | - ReadWriteOnce 71 | resources: 72 | requests: 73 | storage: {{ .Values.swarm.replicas.storage.size }} 74 | -------------------------------------------------------------------------------- /kubernetes/terraform/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | source = "hashicorp/aws" 5 | version = "~> 5.40" 6 | } 7 | helm = { 8 | source = "hashicorp/helm" 9 | version = "~> 2.12" 10 | } 11 | kubernetes = { 12 | source = "hashicorp/kubernetes" 13 | version = "~> 2.38" 14 | } 15 | } 16 | } 17 | 18 | locals { 19 | region = "us-west-2" 20 | } 21 | 22 | provider "aws" { 23 | # https://registry.terraform.io/providers/hashicorp/aws/latest/docs 24 | region = local.region 25 | } 26 | 27 | module "eks_clickhouse" { 28 | source = "github.com/Altinity/terraform-aws-eks-clickhouse" 29 | 30 | # Do not install Kubernetes operator or sample ClickHouse cluster. 31 | install_clickhouse_operator = false 32 | install_clickhouse_cluster = false 33 | 34 | # Set to true if you want to use a public load balancer (and expose ports to the public Internet) 35 | clickhouse_cluster_enable_loadbalancer = false 36 | 37 | eks_cluster_name = "my-eks-cluster" 38 | eks_region = local.region 39 | eks_cidr = "10.0.0.0/16" 40 | eks_cluster_version = "1.32" 41 | 42 | eks_availability_zones = [ 43 | "${local.region}a" 44 | , "${local.region}b" 45 | , "${local.region}c" 46 | ] 47 | eks_private_cidr = [ 48 | "10.0.4.0/22" 49 | , "10.0.8.0/22" 50 | , "10.0.12.0/22" 51 | ] 52 | eks_public_cidr = [ 53 | "10.0.100.0/22" 54 | , "10.0.104.0/22" 55 | , "10.0.108.0/22" 56 | ] 57 | 58 | eks_node_pools = [ 59 | { 60 | name = "clickhouse" 61 | instance_type = "m6i.large" 62 | desired_size = 0 63 | max_size = 10 64 | min_size = 0 65 | zones = ["${local.region}a"] 66 | # zones = ["${local.region}a", "${local.region}b", "${local.region}c"] 67 | #taints = [{ 68 | # key = "antalya" 69 | # value = "clickhouse" 70 | # effect = "NO_SCHEDULE" 71 | #}] 72 | }, 73 | { 74 | name = "clickhouse-swarm" 75 | instance_type = "m6i.xlarge" 76 | desired_size = 0 77 | max_size = 20 78 | min_size = 0 79 | zones = ["${local.region}a"] 80 | #zones = ["${local.region}a", "${local.region}b"] 81 | # zones = ["${local.region}a", "${local.region}b", "${local.region}c"] 82 | taints = [{ 83 | key = "antalya" 84 | value = "swarm" 85 | effect = "NO_SCHEDULE" 86 | }] 87 | }, 88 | { 89 | name = "system" 90 | instance_type = "t3.large" 91 | desired_size = 1 92 | max_size = 10 93 | min_size = 0 94 | zones = ["${local.region}a"] 95 | } 96 | ] 97 | 98 | eks_tags = { 99 | CreatedBy = "antalya-test" 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /kubernetes/manifests/swarm.yaml: -------------------------------------------------------------------------------- 1 | # Swarm cluster with fast EBS volumes for storage. 2 | apiVersion: "clickhouse.altinity.com/v1" 3 | kind: "ClickHouseInstallation" 4 | metadata: 5 | name: "swarm" 6 | spec: 7 | configuration: 8 | clusters: 9 | - name: "example" 10 | layout: 11 | replicas: 12 | - templates: 13 | podTemplate: replica 14 | volumeClaimTemplate: storage 15 | shardsCount: 4 16 | zookeeper: 17 | nodes: 18 | - host: keeper-keeper 19 | port: 2181 20 | users: 21 | cache_enabled/networks/ip: "::/0" 22 | cache_enabled/password: topsecret 23 | cache_enabled/profile: default 24 | profiles: 25 | default/enable_filesystem_cache: 0 26 | #default/enable_filesystem_cache_log: 1 27 | default/filesystem_cache_name: "s3_parquet_cache" 28 | cache_enabled/enable_filesystem_cache: 1 29 | #cache_enabled/enable_filesystem_cache_log: 1 30 | cache_enabled/filesystem_cache_name: "s3_parquet_cache" 31 | files: 32 | config.d/remote_servers.xml: | 33 | 34 | 1 35 | 36 | 37 | 38 | 39 | /clickhouse/discovery/swarm 40 | secret_key 41 | 42 | 43 | 44 | 45 | config.d/filesystem_cache.xml: | 46 | 47 | 48 | 49 | /var/lib/clickhouse/s3_parquet_cache 50 | 50Gi 51 | 52 | 53 | 54 | templates: 55 | podTemplates: 56 | - name: replica 57 | spec: 58 | nodeSelector: 59 | node.kubernetes.io/instance-type: m6i.xlarge 60 | tolerations: 61 | # Run on clickhouse nodes that are dedicated to swarm servers. 62 | - key: "antalya" 63 | operator: "Equal" 64 | value: "swarm" 65 | effect : "NoSchedule" 66 | - key: "dedicated" 67 | operator: "Equal" 68 | value: "clickhouse" 69 | effect : "NoSchedule" 70 | affinity: 71 | # Specify Pod anti-affinity to Pods with the same label "/app" on the same "hostname" 72 | podAntiAffinity: 73 | requiredDuringSchedulingIgnoredDuringExecution: 74 | - labelSelector: 75 | matchExpressions: 76 | - key: "clickhouse.altinity.com/app" 77 | operator: In 78 | values: 79 | - "chop" 80 | topologyKey: "kubernetes.io/hostname" 81 | containers: 82 | - name: clickhouse 83 | image: altinity/clickhouse-server:25.8.9.20207.altinityantalya 84 | volumeClaimTemplates: 85 | - name: storage 86 | # Uncomment for prod systems. You will then need to delete PVCs manually. 87 | # reclaimPolicy: Retain 88 | spec: 89 | storageClassName: gp3-encrypted 90 | accessModes: 91 | - ReadWriteOnce 92 | resources: 93 | requests: 94 | storage: 75Gi 95 | -------------------------------------------------------------------------------- /docker/iceberg_setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Python script showing how to create and populate a table in Iceberg. 3 | 4 | # Uncomment to see the module search path. 5 | # import sys 6 | # print(sys.path) 7 | 8 | from datetime import datetime 9 | 10 | import pyarrow 11 | import pyiceberg 12 | # Allows us to connect to the catalog. 13 | from pyiceberg.catalog import load_catalog 14 | # These are used to create the table structure. 15 | from pyiceberg.schema import Schema 16 | from pyiceberg.types import ( 17 | TimestampType, 18 | FloatType, 19 | DoubleType, 20 | StringType, 21 | NestedField, 22 | ) 23 | from pyiceberg.partitioning import PartitionSpec, PartitionField 24 | from pyiceberg.transforms import DayTransform 25 | from pyiceberg.table.sorting import SortOrder, SortField 26 | from pyiceberg.transforms import IdentityTransform 27 | 28 | print("Connect to the catalog") 29 | catalog = load_catalog( 30 | "rest", 31 | **{ 32 | "uri": "http://localhost:5000/", # REST server URL. 33 | "type": "rest", 34 | "token": "foo", 35 | "s3.endpoint": f"http://localhost:9002", # Minio URI and credentials 36 | "s3.access-key-id": "minio", 37 | "s3.secret-access-key": "minio123", 38 | } 39 | ) 40 | 41 | # Set up a namespace if it does not exist. 42 | print("Create namespace iceberg") 43 | try: 44 | catalog.create_namespace("iceberg") 45 | print("--Created") 46 | except pyiceberg.exceptions.NamespaceAlreadyExistsError: 47 | print("--Already exists") 48 | 49 | print("List namespaces") 50 | ns_list = catalog.list_namespaces() 51 | for ns in ns_list: 52 | print(ns) 53 | 54 | # List tables and delete the bids table if it exists. 55 | print("List tables") 56 | tab_list = catalog.list_tables("iceberg") 57 | for tab in tab_list: 58 | print(tab, type(tab)) 59 | if tab[0] == "iceberg" and tab[1] == "bids": 60 | print("Dropping bids table") 61 | catalog.drop_table("iceberg.bids") 62 | 63 | # Now create the test table. It's partitioned by datetime and 64 | # sorted by symbol. 65 | schema = Schema( 66 | NestedField(field_id=1, name="datetime", field_type=TimestampType(), required=False), 67 | NestedField(field_id=2, name="symbol", field_type=StringType(), required=False), 68 | NestedField(field_id=3, name="bid", field_type=DoubleType(), required=False), 69 | NestedField(field_id=4, name="ask", field_type=DoubleType(), required=False), 70 | ) 71 | partition_spec = PartitionSpec( 72 | PartitionField( 73 | source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" 74 | ) 75 | ) 76 | sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform())) 77 | table = catalog.create_table( 78 | identifier="iceberg.bids", 79 | schema=schema, 80 | location="s3://warehouse/data", 81 | partition_spec=partition_spec, 82 | sort_order=sort_order, 83 | ) 84 | 85 | # Define a helper function to create datetime values. 86 | def to_dt(string): 87 | format = "%Y-%m-%d %H:%M:%S" 88 | dt = datetime.strptime(string, format) 89 | return dt 90 | 91 | # Generate some trading data. Of course we use AAPL as an example. 92 | print("Add some data") 93 | import pyarrow as pa 94 | tt = pa.timestamp('us') 95 | df = pa.Table.from_pylist( 96 | [ 97 | {"datetime": pa.scalar(to_dt("2019-08-07 08:35:00"), tt), "symbol": "AAPL", "bid": 195.23, "ask": 195.28}, 98 | {"datetime": pa.scalar(to_dt("2019-08-07 08:35:00"), tt), "symbol": "AAPL", "bid": 195.22, "ask": 195.28}, 99 | ], 100 | ) 101 | table.append(df) 102 | 103 | # Add more trading data on another day. This will be in another partiion. 104 | print("Add some more data") 105 | df2 = pa.Table.from_pylist( 106 | [ 107 | {"datetime": pa.scalar(to_dt("2019-08-09 08:35:00"), tt), "symbol": "AAPL", "bid": 198.23, "ask": 195.45}, 108 | {"datetime": pa.scalar(to_dt("2019-08-09 08:35:00"), tt), "symbol": "AAPL", "bid": 198.25, "ask": 198.50}, 109 | ], 110 | ) 111 | table.append(df2) 112 | -------------------------------------------------------------------------------- /docs/concepts.md: -------------------------------------------------------------------------------- 1 | # Antalya Concepts Guide 2 | 3 | ## Overview 4 | 5 | Project Antalya is an extended version of ClickHouse that offers all 6 | existing capabilities plus additional features that allow Antalya clusters 7 | to use Iceberg as shared storage. The following diagram shows the main 8 | parts of an Antalya installation. 9 | 10 | ![Antalya Reference Architecture](images/Antalya-Reference-Architecture-2025-02-17.png) 11 | 12 | ## ClickHouse Compatibility 13 | 14 | Antalya builds are based on upstream ClickHouse and follow ClickHouse 15 | versioning. They are drop-in replacements for the matching ClickHouse 16 | version. They are built using the same CI/CD pipelines as Altinity 17 | Stable Builds. 18 | 19 | ## Iceberg, Parquet, and Object Storage 20 | 21 | Antalya includes extended support for fast operation on Iceberg 22 | data lakes using Parquet data files on S3 compatible storage. Antalya 23 | extensions include the following: 24 | 25 | * Integration with Iceberg REST catalog (compatible with upstream) 26 | * Parquet bloom filter support (compatible with upstream) 27 | * Iceberg partition pruning (compatible with upstream) 28 | * Parquet file metadata cache 29 | * Boolean and int type support on native Parquet reader (compatible with upstream) 30 | 31 | ## Iceberg Specification Support 32 | 33 | Generally speaking Antalya Iceberg support matches upstream ClickHouse. 34 | Antalya supports reading Iceberg V2. It cannot write to Iceberg 35 | tables. For that you must currently use other tools, such as Spark 36 | or pyiceberg. 37 | 38 | There are a number of bugs and missing features in Iceberg support. If you 39 | find something unexpected, please 40 | [log an issue](https://github.com/Altinity/ClickHouse/issues/new/choose) 41 | on the Altinity ClickHouse repo. Use one of the Project Antalya issue 42 | templates so that the report is automatically tagged to Project Antalya. 43 | 44 | ### Iceberg Database Engine 45 | 46 | The Iceberg database engine encapsulates the tables in a single Iceberg 47 | REST catalog. REST catalogs enumerate the metadata for Iceberg tables, 48 | and the database engine makes them look like ClickHouse tables. This is 49 | the most natural way to interate with Iceberg tables. 50 | 51 | ### Iceberg Table Engine and Table Function 52 | 53 | Antalya offers Iceberg table engine and functions just like upstream 54 | ClickHouse. They encapsulate a single table using the object storage 55 | path to locate the table metadata and data. Currently only one table 56 | can use the path. 57 | 58 | ### Hive and Plain S3 59 | 60 | Antalya can also read Parquet data directly from S3 as well as Hive 61 | format. The capabilities are largely identical to upstream ClickHouse. 62 | 63 | ## Swarm Clusters 64 | 65 | Antalya introduces the notion of swarm clusters, which are clusters of 66 | stateless ClickHouse servers that can be used for parallel query as 67 | well as (in future) writes to Iceberg. Swarm clusters can scale up and 68 | down quickly. 69 | 70 | To use a swarm cluster you must first provision at least one Antalya 71 | server to act as a query initiator. This server must have access to the 72 | table schema, for example by connecting to an Iceberg database using the 73 | `CREATE DATABASE ... Engine=Iceberg` command. 74 | 75 | You can dispatch a query on S3 files or Iceberg tables to a swarm 76 | cluster by adding the `object_storage_cluster = ` 77 | setting to the query. You can also set this value in a profile or as as 78 | session setting. 79 | 80 | The Antalya initiator will parse the query, then dispatch subqueries to 81 | nodes of the swarm for query on individual Parquet files. The results 82 | are streamed back to the initiator, which merges them and returns final 83 | results to the client application. 84 | 85 | ## Swarm Auto-Discovery using Keeper 86 | 87 | Antalya uses Keeper servers to implement swarm cluster auto-discovery. 88 | 89 | 1. Each swarm cluster can register itself in one or more clusters. Each 90 | cluster is registered on a unique path in Keeper. 91 | 92 | 2. Initiators read cluster definitions from Keeper. They are updated as 93 | the cluster grows or shrinks. 94 | 95 | Antalya also supports the notion of an auxiliary Keeper server for cluster 96 | discovery. This means that Antalya clusters can use one Keeper ensemble 97 | to control replication, and another Keeper server for auto-discovery. 98 | 99 | Swarm clusters do not use replication. They only need Keeper for 100 | auto-discovery. 101 | 102 | ## Tiered Storage between MergeTree and Iceberg 103 | 104 | Antalya will provide tiered storage between MergeTree and Iceberg tables. 105 | Tiered storage includes the following features. 106 | 107 | 1. ALTER TABLE MOVE command to move parts from MergeTree to external 108 | Iceberg tables. 109 | 2. TTL MOVE to external Iceberg table. Works as current tiered storage but will 110 | also permit different partitioning and sort orders in Iceberg. 111 | 3. Transparent reads across tiered MergeTree / Iceberg tables. 112 | 113 | ## Runtime Environment 114 | 115 | Antalya servers can run anywhere ClickHouse does now. Cloud native 116 | operation on Kubernetes provides a portable and easy-to-configure path 117 | for scaling swarm servers. You can also run Antalya clusters on bare 118 | metal servers and VMs, just as ClickHouse does today. 119 | 120 | ## Future Roadmap 121 | 122 | Antalya has an active roadmap. Here are some of the planned features. 123 | 124 | * Extension of Altinity Backup for ClickHouse to support Antalya servers 125 | with Iceberg external tables. 126 | * Automatic archiving of level 0 parts to Iceberg so that all data is 127 | visible from the time of ingest. 128 | * Materialized views on Iceberg tables. 129 | * Fast ingest using Swarm server. This will amortize the effort of generating 130 | Parquet files using cheap compute servers. 131 | 132 | There are many more possibilities. We're looking for contributors. Join 133 | the fun! 134 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | vector: 3 | depends_on: 4 | - minio-init 5 | - keeper 6 | image: altinity/clickhouse-server:25.8.9.20238.altinityantalya 7 | container_name: vector 8 | hostname: vector 9 | ports: 10 | - 8123:8123 11 | - 9000:9000 12 | volumes: 13 | - ./config.d/vector:/etc/clickhouse-server/config.d 14 | - ./users.d:/etc/clickhouse-server/users.d 15 | - ./data/vector/log:/var/log/clickhouse-server 16 | - ./data/vector/clickhouse:/var/lib/clickhouse 17 | environment: 18 | - AWS_ACCESS_KEY_ID=minio 19 | - AWS_SECRET_ACCESS_KEY=minio123 20 | - AWS_REGION=minio 21 | keeper: 22 | image: altinity/clickhouse-keeper:25.8.9.20238.altinityantalya 23 | container_name: keeper 24 | hostname: keeper 25 | ports: 26 | - 9181:9181 27 | volumes: 28 | # Note special location of Keeper config overrides. 29 | - ./config.d/keeper:/etc/clickhouse-keeper/keeper_config.d 30 | - ./data/keeper/log:/var/log/clickhouse-keeper 31 | - ./data/keeper/keeper:/var/lib/keeper 32 | swarm-1: 33 | image: altinity/clickhouse-server:25.8.9.20238.altinityantalya 34 | container_name: swarm-1 35 | hostname: swarm-1 36 | volumes: 37 | - ./config.d/swarm:/etc/clickhouse-server/config.d 38 | - ./users.d:/etc/clickhouse-server/users.d 39 | - ./data/swarm-1/log:/var/log/clickhouse-server 40 | - ./data/swarm-1/clickhouse:/var/lib/clickhouse 41 | environment: 42 | - AWS_ACCESS_KEY_ID=minio 43 | - AWS_SECRET_ACCESS_KEY=minio123 44 | - AWS_REGION=minio 45 | swarm-2: 46 | image: altinity/clickhouse-server:25.8.9.20238.altinityantalya 47 | container_name: swarm-2 48 | hostname: swarm-2 49 | volumes: 50 | - ./config.d/swarm:/etc/clickhouse-server/config.d 51 | - ./users.d:/etc/clickhouse-server/users.d 52 | - ./data/swarm-2/log:/var/log/clickhouse-server 53 | - ./data/swarm-2/clickhouse:/var/lib/clickhouse 54 | environment: 55 | - AWS_ACCESS_KEY_ID=minio 56 | - AWS_SECRET_ACCESS_KEY=minio123 57 | - AWS_REGION=minio 58 | minio: 59 | image: minio/minio:RELEASE.2025-03-12T18-04-18Z 60 | container_name: minio 61 | volumes: 62 | - ./data/minio/data:/data 63 | environment: 64 | - MINIO_ROOT_USER=minio 65 | - MINIO_ROOT_PASSWORD=minio123 66 | - MINIO_DOMAIN=minio 67 | networks: 68 | default: 69 | aliases: 70 | - warehouse.minio 71 | ports: 72 | - 9001:9001 73 | - 9002:9000 74 | command: ["server", "/data", "--console-address", ":9001"] 75 | minio-init: 76 | depends_on: 77 | - minio 78 | image: minio/mc:RELEASE.2025-03-12T17-29-24Z 79 | container_name: mc 80 | environment: 81 | - AWS_ACCESS_KEY_ID=minio 82 | - AWS_SECRET_ACCESS_KEY=minio123 83 | - AWS_REGION=minio 84 | entrypoint: > 85 | /bin/sh -c " 86 | until (/usr/bin/mc alias set minio http://minio:9000 minio minio123) do echo '...waiting...' && sleep 1; done; 87 | /usr/bin/mc mb minio/warehouse --ignore-existing; 88 | /usr/bin/mc policy set public minio/warehouse; 89 | tail -f /dev/null 90 | " 91 | ice-rest-catalog: 92 | image: altinity/ice-rest-catalog:${ICE_REST_CATALOG_TAG:-latest} 93 | pull_policy: ${ICE_REST_CATALOG_PULL_POLICY:-always} 94 | restart: unless-stopped 95 | ports: 96 | - 5000:5000 # iceberg/http 97 | configs: 98 | - source: ice-rest-catalog-yaml 99 | target: /etc/ice/ice-rest-catalog.yaml 100 | volumes: 101 | # for access to /var/lib/ice-rest-catalog/db.sqlite 102 | - ./data/ice-rest-catalog/var/lib/ice-rest-catalog:/var/lib/ice-rest-catalog 103 | depends_on: 104 | - minio-init 105 | spark-iceberg: 106 | # https://github.com/databricks/docker-spark-iceberg 107 | image: tabulario/spark-iceberg:3.5.5_1.8.1 108 | container_name: spark-iceberg 109 | # volumes: 110 | # - ./notebooks:/home/iceberg/notebooks/local 111 | configs: 112 | - source: spark-defaults.conf 113 | target: /opt/spark/conf/spark-defaults.conf 114 | ports: 115 | - 8888:8888 # jupyter-notebook 116 | - 8080:8080 # spark-master 117 | configs: 118 | ice-rest-catalog-yaml: 119 | content: | 120 | uri: jdbc:sqlite:file:/var/lib/ice-rest-catalog/db.sqlite?journal_mode=WAL&synchronous=OFF&journal_size_limit=500 121 | #warehouse: s3://iceberg_data 122 | warehouse: s3://warehouse 123 | s3: 124 | endpoint: http://minio:9000 125 | pathStyleAccess: true 126 | accessKeyID: minio 127 | secretAccessKey: minio123 128 | region: minio 129 | bearerTokens: 130 | - value: foo 131 | spark-defaults.conf: 132 | content: | 133 | spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions 134 | spark.sql.catalog.default org.apache.iceberg.spark.SparkCatalog 135 | spark.sql.catalog.default.type rest 136 | spark.sql.catalog.default.uri http://ice-rest-catalog:5000 137 | spark.sql.catalog.default.header.authorization bearer foo 138 | spark.sql.catalog.default.io-impl org.apache.iceberg.aws.s3.S3FileIO 139 | spark.sql.catalog.default.warehouse s3://warehouse 140 | spark.sql.catalog.default.s3.endpoint http://minio:9000 141 | spark.sql.catalog.default.s3.path-style-access true 142 | spark.sql.catalog.default.s3.access-key minio 143 | spark.sql.catalog.default.s3.secret-key minio123 144 | spark.sql.catalog.default.client.region minio 145 | spark.sql.catalog.default.s3.ssl-enabled false 146 | spark.sql.defaultCatalog default 147 | spark.eventLog.enabled true 148 | spark.eventLog.dir /home/iceberg/spark-events 149 | spark.history.fs.logDirectory /home/iceberg/spark-events 150 | spark.sql.catalogImplementation in-memory 151 | # spark.log.level DEBUG 152 | -------------------------------------------------------------------------------- /kubernetes/manifests/nvme/local-storage-eks-nvme-ssd.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Source: local-static-provisioner/templates/serviceaccount.yaml 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: local-static-provisioner 7 | namespace: antalya 8 | labels: 9 | helm.sh/chart: local-static-provisioner-2.0.0 10 | app.kubernetes.io/name: local-static-provisioner 11 | app.kubernetes.io/managed-by: Helm 12 | app.kubernetes.io/instance: local-static-provisioner 13 | --- 14 | # Source: local-static-provisioner/templates/configmap.yaml 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: local-static-provisioner-config 19 | namespace: antalya 20 | labels: 21 | helm.sh/chart: local-static-provisioner-2.0.0 22 | app.kubernetes.io/name: local-static-provisioner 23 | app.kubernetes.io/managed-by: Helm 24 | app.kubernetes.io/instance: local-static-provisioner 25 | data: 26 | storageClassMap: | 27 | nvme-ssd: 28 | hostDir: /pv-disks 29 | mountDir: /pv-disks 30 | #hostDir: /dev/disk/kubernetes 31 | #mountDir: /dev/disk/kubernetes 32 | --- 33 | # Source: local-static-provisioner/templates/storageclass.yaml 34 | apiVersion: storage.k8s.io/v1 35 | kind: StorageClass 36 | metadata: 37 | name: nvme-ssd 38 | labels: 39 | helm.sh/chart: local-static-provisioner-2.0.0 40 | app.kubernetes.io/name: local-static-provisioner 41 | app.kubernetes.io/managed-by: Helm 42 | app.kubernetes.io/instance: local-static-provisioner 43 | #provisioner: csi-mock-driver 44 | provisioner: dummy 45 | volumeBindingMode: WaitForFirstConsumer 46 | reclaimPolicy: Delete 47 | --- 48 | # Source: local-static-provisioner/templates/rbac.yaml 49 | apiVersion: rbac.authorization.k8s.io/v1 50 | kind: ClusterRole 51 | metadata: 52 | name: local-static-provisioner-node-clusterrole 53 | labels: 54 | helm.sh/chart: local-static-provisioner-2.0.0 55 | app.kubernetes.io/name: local-static-provisioner 56 | app.kubernetes.io/managed-by: Helm 57 | app.kubernetes.io/instance: local-static-provisioner 58 | rules: 59 | - apiGroups: [""] 60 | resources: ["persistentvolumes"] 61 | verbs: ["get", "list", "watch", "create", "delete"] 62 | - apiGroups: ["storage.k8s.io"] 63 | resources: ["storageclasses"] 64 | verbs: ["get", "list", "watch"] 65 | - apiGroups: [""] 66 | resources: ["events"] 67 | verbs: ["watch"] 68 | - apiGroups: ["", "events.k8s.io"] 69 | resources: ["events"] 70 | verbs: ["create", "update", "patch"] 71 | - apiGroups: [""] 72 | resources: ["nodes"] 73 | verbs: ["get"] 74 | --- 75 | # Source: local-static-provisioner/templates/rbac.yaml 76 | apiVersion: rbac.authorization.k8s.io/v1 77 | kind: ClusterRoleBinding 78 | metadata: 79 | name: local-static-provisioner-node-binding 80 | labels: 81 | helm.sh/chart: local-static-provisioner-2.0.0 82 | app.kubernetes.io/name: local-static-provisioner 83 | app.kubernetes.io/managed-by: Helm 84 | app.kubernetes.io/instance: local-static-provisioner 85 | subjects: 86 | - kind: ServiceAccount 87 | name: local-static-provisioner 88 | namespace: antalya 89 | roleRef: 90 | kind: ClusterRole 91 | name: local-static-provisioner-node-clusterrole 92 | apiGroup: rbac.authorization.k8s.io 93 | --- 94 | # Source: local-static-provisioner/templates/daemonset_linux.yaml 95 | apiVersion: apps/v1 96 | kind: DaemonSet 97 | metadata: 98 | name: local-static-provisioner 99 | namespace: antalya 100 | labels: 101 | helm.sh/chart: local-static-provisioner-2.0.0 102 | app.kubernetes.io/name: local-static-provisioner 103 | app.kubernetes.io/managed-by: Helm 104 | app.kubernetes.io/instance: local-static-provisioner 105 | spec: 106 | selector: 107 | matchLabels: 108 | app.kubernetes.io/name: local-static-provisioner 109 | app.kubernetes.io/instance: local-static-provisioner 110 | template: 111 | metadata: 112 | labels: 113 | app.kubernetes.io/name: local-static-provisioner 114 | app.kubernetes.io/instance: local-static-provisioner 115 | annotations: 116 | checksum/config: b110cd9aea997eefa707c673fe5efa712280f77f1e5af3ff591359246cba3d9e 117 | spec: 118 | hostPID: false 119 | serviceAccountName: local-static-provisioner 120 | nodeSelector: 121 | aws.amazon.com/eks-local-ssd: "true" 122 | # Run on clickhouse nodes that are dedicated to swarm servers. 123 | tolerations: 124 | - key: "antalya" 125 | operator: "Equal" 126 | value: "nvme-swarm" 127 | effect : "NoSchedule" 128 | - key: "dedicated" 129 | operator: "Equal" 130 | value: "clickhouse" 131 | effect : "NoSchedule" 132 | containers: 133 | - name: provisioner 134 | image: registry.k8s.io/sig-storage/local-volume-provisioner:v2.7.0 135 | securityContext: 136 | privileged: true 137 | env: 138 | - name: MY_NODE_NAME 139 | valueFrom: 140 | fieldRef: 141 | fieldPath: spec.nodeName 142 | - name: MY_NAMESPACE 143 | valueFrom: 144 | fieldRef: 145 | fieldPath: metadata.namespace 146 | - name: JOB_CONTAINER_IMAGE 147 | value: registry.k8s.io/sig-storage/local-volume-provisioner:v2.7.0 148 | ports: 149 | - name: metrics 150 | containerPort: 8080 151 | volumeMounts: 152 | - name: provisioner-config 153 | mountPath: /etc/provisioner/config 154 | readOnly: true 155 | - name: provisioner-dev 156 | mountPath: /dev 157 | - name: nvme-ssd 158 | #mountPath: /dev/disk/kubernetes 159 | mountPath: /pv-disks 160 | mountPropagation: HostToContainer 161 | volumes: 162 | - name: provisioner-config 163 | configMap: 164 | name: local-static-provisioner-config 165 | - name: provisioner-dev 166 | hostPath: 167 | path: /dev 168 | - name: nvme-ssd 169 | hostPath: 170 | #path: /dev/disk/kubernetes 171 | path: /pv-disks 172 | -------------------------------------------------------------------------------- /kubernetes/ice/README.md: -------------------------------------------------------------------------------- 1 | # Configuring an Ice REST catalog in Kubernetes 2 | 3 | Iceberg REST catalogs enable ClickHouse to access Iceberg catalogs as if 4 | they were databases. This directory show how to leverage the Altinity 5 | [Ice Toolset](https://github.com/Altinity/ice) to add an Iceberg REST 6 | catalog to an existing EKS cluster. 7 | 8 | ## Prerequisites 9 | 10 | You should have an EKS cluster already setup using the Terraform 11 | [main.tf](../kubernetes/main.tf) script. This procedure will also 12 | work for EKS clusters that you create by other means. 13 | 14 | You will also need eksctl. Install it following the 15 | [eksctl installation instructions](https://eksctl.io/installation/). 16 | 17 | Finally, these examples assume you have an antalya namespace that is 18 | also the default. 19 | 20 | ## IAM Configuration 21 | 22 | Start by ensuring that OIDC is correctly configured in your cluster. 23 | Run the following command. You should see a hex string like 24 | CA23823F2D578D6A905B5718679C69D2 as output. If you don't see it refer 25 | to [AWS EKS OIDC docs](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) 26 | for help. 27 | 28 | ``` 29 | aws eks describe-cluster --name my-eks-cluster \ 30 | --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5 31 | ``` 32 | 33 | Next, use eksctl to create a Kubernetes ServiceAccount that has 34 | privileges to read and write to S3 buckets. The attached policy is 35 | suitable for testing only--it gives read/write access to all buckets. 36 | In a production environment you would attach a more restrictive policy 37 | that gives privileges on certain buckets. 38 | 39 | ``` 40 | eksctl create iamserviceaccount --name ice-rest-catalog \ 41 | --namespace antalya \ 42 | --cluster my-eks-cluster --role-name eksctl-cluster-autoscaler-role \ 43 | --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3FullAccess \ 44 | --approve 45 | ``` 46 | 47 | If you get something wrong and it fails, you need to clean up. Use the 48 | following commands. It's important to clean up *everything* to avoid 49 | weird failures when you try again. 50 | 51 | ``` 52 | eksctl delete iamserviceaccount --name ice-rest-catalog \ 53 | --namespace antalya --cluster my-eks-cluster 54 | aws iam detach-role-policy --role-name=eksctl-cluster-autoscaler-role \ 55 | --policy-arn=arn:aws:iam::aws:policy/AmazonS3FullAccess 56 | aws iam delete-role --role-name=eksctl-cluster-autoscaler-role 57 | ``` 58 | 59 | If it still does not work, try going into CloudFormation and deleting 60 | the stack that created the ServiceAccount. (We figured this out so you 61 | don't have to. ;) 62 | 63 | ## REST Catalog Start and Data Loading 64 | 65 | For the remainder of the setup use the 66 | [Altinity ice project eks sample](https://github.com/Altinity/ice/tree/master/examples/eks). 67 | 68 | Clone the ice project to get started. 69 | ``` 70 | git clone https://github.com/Altinity/ice 71 | cd ice/examples/eks 72 | ``` 73 | 74 | Follow an adapted procedure from the local README.md. Skip the `eksctl 75 | create cluster` command as you already have a cluster set up. You'll 76 | need to [install devbox](https://github.com/jetify-com/devbox) if you 77 | don't already have it. This example uses us-west-2 as the AWS region. 78 | 79 | ``` 80 | devbox shell 81 | 82 | export CATALOG_BUCKET="$USER-ice-rest-catalog-demo" 83 | export AWS_REGION=us-west-2 84 | 85 | # create bucket if you don't have it already. 86 | aws s3api create-bucket --bucket "$CATALOG_BUCKET" \ 87 | --create-bucket-configuration "LocationConstraint=$AWS_REGION" 88 | 89 | # deploy etcd 90 | kubectl -n antalya apply -f etcd.eks.yaml 91 | 92 | # deploy ice-rest-catalog. The debug-with-ice image has the ice 93 | # utility bundled in the catalog container. 94 | cat ice-rest-catalog.eks.envsubst.yaml |\ 95 | envsubst -no-unset -no-empty |\ 96 | sed -e 's/debug-0.0.0-SNAPSHOT/debug-with-ice/' > ice-rest-catalog.eks.yaml 97 | kubectl -n antalya apply -f ice-rest-catalog.eks.yaml 98 | 99 | # add data to the catalog to make things interesting. 100 | kubectl -n antalya exec -it ice-rest-catalog-0 -- ice insert nyc.taxis -p \ 101 | https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet 102 | 103 | # add a larger dataset from AWS public BTC transaction data. 104 | kubectl -n antalya exec -it ice-rest-catalog-0 -- ice insert btc.transactions \ 105 | -p --s3-no-sign-request --s3-region=us-east-2 \ 106 | 's3://aws-public-blockchain/v1.0/btc/transactions/date=2025-0*-*/*.parquet' 107 | ``` 108 | 109 | ## Adding an SQS notification queue for the S3 bucket. 110 | 111 | Use Terraform to add an SQS queue that receives S3 notifications when new .parquet files are created. 112 | 113 | ```bash 114 | # Use the same CATALOG_BUCKET value from the previous section 115 | export TF_VAR_catalog_bucket=$CATALOG_BUCKET 116 | 117 | terraform init 118 | terraform apply 119 | ``` 120 | 121 | Note the BUCKET and SQS queue URL from the output for configuring the ice command to auto-load 122 | new files in SQL to a table. 123 | ``` 124 | echo $" 125 | export CATALOG_BUCKET=$(terraform output -raw s3_bucket_name) 126 | export CATALOG_SQS_QUEUE_URL=$(terraform output -raw sqs_queue_url) 127 | " > tf.export 128 | 129 | . tf.export 130 | ``` 131 | 132 | You can now use the bucket and catalog names in ice commands like the following: 133 | ``` 134 | ice insert blog.tripdata_watch -p --force-no-copy --skip-duplicates \ 135 | "s3://$CATALOG_BUCKET/ICE_WATCH/blog/tripdata_watch/*.parquet"\ 136 | --watch="$CATALOG_SQS_QUEUE_URL" 137 | ``` 138 | 139 | ## Connecting ClickHouse to the ice catalog. 140 | 141 | Issue SQL commands to connect to the Ice catalog. Adjust the string in the warehouse setting to 142 | match the name of your S3 bucket. 143 | 144 | ``` 145 | kubectl -n antalya exec -it chi-vector-example-0-0-0 -- clickhouse-client 146 | ... 147 | SET allow_experimental_database_iceberg = 1; 148 | 149 | -- (re)create iceberg db 150 | DROP DATABASE IF EXISTS ice; 151 | 152 | CREATE DATABASE ice 153 | ENGINE = DataLakeCatalog('http://ice-rest-catalog:5000') 154 | SETTINGS catalog_type = 'rest', 155 | auth_header = 'Authorization: Bearer foo', 156 | warehouse = 's3://-ice-rest-catalog-demo'; 157 | 158 | SHOW TABLES FROM ice; 159 | ``` 160 | 161 | The last command should show the tables you just added. 162 | 163 | # Troubleshooting 164 | 165 | The ice toolset is pretty easy to set up. If you run into trouble it's likely something 166 | to do with AWS IAM. For more hints check out the ice 167 | [examples/eks/README.md](https://github.com/Altinity/ice/tree/master/examples/eks#readme). 168 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | AltinityDB Slack 5 | 6 | 7 | 8 | 9 | 10 | Altinity company logo 11 | 12 | 13 |
14 | 15 | # Altinity Project Antalya Examples 16 | 17 | Project Antalya is a new branch of ClickHouse® code designed to 18 | integrate real-time analytic query with data lakes. This project 19 | provides documentation as well as working code examples to help you use 20 | and contribute to Antalya. 21 | 22 | *Important Note!* Altinity maintains and supports Project Antalya. Altinity 23 | is not affiliated or associated with ClickHouse Inc any way. ClickHouse® is 24 | a registered trademark of ClickHouse, Inc. 25 | 26 | See the 27 | [Community Support Section](#community-support) if you want to ask 28 | questions or log ideas and issues. 29 | 30 | ## Project Antalya Goals 31 | 32 | Analytic data size has increased to a point where traditional designs 33 | based on shared-nothing architectures and block storage are prohibitively 34 | expensive to operate. Project Antalya introduces a fully open source 35 | architecture for cost-efficient, real-time systems using cheap object 36 | storage and scalable compute, specifically: 37 | 38 | * Enable real-time analytics to work off a single copy of 39 | data that is shared with AI and data science applications. 40 | * Provide a single SQL endpoint for native ClickHouse® and data lake data. 41 | * Use open table formats to enable easy access from any application type. 42 | * Separate compute and storage; moreover, allow users to scale compute 43 | for ingest, merge, transformation, and query independently. 44 | 45 | Antalya will implement these goals through the following concrete features: 46 | 47 | 1. Optimize query performance of ClickHouse® on Parquet files stored 48 | S3-compatible object storage. 49 | 2. Enable ClickHouse® clusters to add pools of stateless servers aka swarm 50 | clusters that handle query and insert operations on shared object storage 51 | files with linear scaling. 52 | 3. Adapt ClickHouse® to use Iceberg tables as shared storage. 53 | 4. Enable ClickHouse® clusters to extend existing tables onto unlimited 54 | Iceberg storage with transparent query across both native MergeTree and 55 | Parquet data. 56 | 5. Simplify backup and DR by leveraging Iceberg features like snapshots. 57 | 6. Maintain full compability with upstream ClickHouse® features and 58 | bug fixes. 59 | 60 | ## Roadmap 61 | 62 | [Project Antalya Roadmap 2025 - Real-Time Data Lakes](https://github.com/Altinity/ClickHouse/issues/804) 63 | 64 | ## Licensing 65 | 66 | Project Antalya code is licensed under Apache 2.0 license. There are no feature 67 | hold-backs. 68 | 69 | ## Quick Start 70 | 71 | See the [Docker Quick Start](./docker/README.md) to try out Antalya in 72 | a few minutes using Docker Compose on a laptop. 73 | 74 | ## Scalable Swarm Example 75 | 76 | For a fully functional swarm cluster implemention, look at the 77 | [kubernetes](kubernetes/README.md) example. It demonstrates use of swarm 78 | clusters on a large blockchain dataset stored in Parquet. 79 | 80 | ## Project Antalya Binaries 81 | 82 | ### Packages 83 | 84 | Project Antalya ClickHouse® server and keeper packages are available on the 85 | [builds.altinity.cloud](https://builds.altinity.cloud/) page. Scan to the last 86 | section to find them. 87 | 88 | ### Containers 89 | 90 | Project Antalya ClickHouse® server and ClickHouse® keeper containers 91 | are available on Docker Hub. 92 | 93 | Check for the latest build on 94 | [Docker Hub](https://hub.docker.com/r/altinity/clickhouse-server/tags). 95 | 96 | ## Documentation 97 | 98 | Look in the docs directory for current documentation. More is on the way. 99 | 100 | * [Project Antalya Concepts Guide](docs/concepts.md) 101 | * [Command and Configuration Reference](docs/reference.md) 102 | 103 | See also the [Project Antalya Launch Video](https://altinity.com/events/scale-clickhouse-queries-infinitely-with-10x-cheaper-storage-introducing-project-antalya) 104 | for an introduction to Project Antalya and a demo of performance. 105 | 106 | The [Altinity Blog](https://altinity.com/blog/) has regular articles 107 | on Project Antalya features and performance. 108 | 109 | ## Code 110 | 111 | To access Project Antalya code run the following commands. 112 | 113 | ``` 114 | git clone git@github.com:Altinity/ClickHouse.git Altinity-ClickHouse 115 | cd Altinity-ClickHouse 116 | git branch 117 | ``` 118 | 119 | You will be in the antalya branch by default. 120 | 121 | ## Building 122 | 123 | Build instructions are located [here](https://github.com/Altinity/ClickHouse/blob/antalya/docs/en/development/developer-instruction.md) 124 | in the Altinity ClickHouse code tree. Project Antalya code does not 125 | introduce new libaries or build procedures. 126 | 127 | ## Contributing 128 | 129 | We welcome contributions. We're setting up procedures for community 130 | contribution. For now, please contact us in Slack to find out how to 131 | join the project. 132 | 133 | ## Community Support 134 | 135 | * Join the [AltinityDB Slack Workspace](https://altinity.com/slack) to ask questions. 136 | * [Log an issue on this documentation](https://github.com/Altinity/antalya-examples/issues). 137 | * [Log an issue on Antalya code](https://github.com/Altinity/ClickHouse/issues). 138 | 139 | ## Commercial Support 140 | 141 | Altinity is the primary maintainer of Project Antalya. It is the 142 | basis of our data lake-enabled Altinity.Cloud and is also used in 143 | self-managed installations. Altinity offers a range of services related 144 | to ClickHouse® and data lakes. 145 | 146 | - [Official website](https://altinity.com/) - Get a high level overview of Altinity and our offerings. 147 | - [Altinity.Cloud](https://altinity.com/cloud-database/) - Run Antalya in your cloud or ours. 148 | - [Altinity Support](https://altinity.com/support/) - Get Enterprise-class support for ClickHouse®. 149 | - [Slack](https://altinity.com/slack) - Talk directly with ClickHouse® users and Altinity devs. 150 | - [Contact us](https://hubs.la/Q020sH3Z0) - Contact Altinity with your questions or issues. 151 | - [Free consultation](https://hubs.la/Q020sHkv0) - Get a free consultation with a ClickHouse® expert today. 152 | -------------------------------------------------------------------------------- /kubernetes/manifests/nvme/README.md: -------------------------------------------------------------------------------- 1 | # Antalya Swarms using NVMe Backed Workers (Experimental) 2 | 3 | This directory shows how to set up an Antalya swarm cluster using 4 | workers with local NVMe. It is still work in progress. 5 | 6 | The current examples show prototype configuration using the following 7 | options: 8 | * hostPath volumes 9 | * local storage volumes 10 | 11 | Examples apply to apply to AWS EKS only. 12 | 13 | ## NVMe SSD provisioning. 14 | 15 | This is a prerequisite for either swarm type. It creates 16 | a daemonset that automatically formats NVMe drives 17 | on new EKS workers. The source code is currently located 18 | [here](https://github.com/hodgesrm/eks-nvme-ssd-provisioner). It will 19 | be transferred to the Altinity org shortly. 20 | 21 | ``` 22 | kubectl apply -f eks-nvme-ssd-provisioner.yaml 23 | ``` 24 | 25 | The daemonset has tolerations necessary to operate on swarm nodes. 26 | Confirm that it is working by checking that a daemon appears on each new 27 | worker node. You should also see log messages when the daemon formats 28 | the file system on a new worker, as shown by the following example. 29 | 30 | ``` 31 | $ kubectl logs eks-nvme-ssd-provisioner-zfmv7 -n kube-system 32 | mke2fs 1.47.0 (5-Feb-2023) 33 | Discarding device blocks: done 34 | Creating filesystem with 228759765 4k blocks and 57196544 inodes 35 | Filesystem UUID: fa6a9dd7-322b-456b-bc86-176c5dee2470 36 | Superblock backups stored on blocks: 37 | 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 38 | 4096000, 7962624, 11239424, 20480000, 23887872, 71663616, 78675968, 39 | 102400000, 214990848 40 | 41 | Allocating group tables: done 42 | Writing inode tables: done 43 | Creating journal (262144 blocks): done 44 | Writing superblocks and filesystem accounting information: done 45 | 46 | Device /dev/nvme1n1 has been mounted to /pv-disks/fa6a9dd7-322b-456b-bc86-176c5dee2470 47 | NVMe SSD provisioning is done and I will go to sleep now 48 | ``` 49 | 50 | The log messages are helpful if you need to debug problems with mount locations. 51 | 52 | ## Swarm using NVMe SSD with hostPath volumes (Experimental) 53 | 54 | The swarm-hostpath.yaml is configured to use 55 | [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) 56 | volumes. 57 | 58 | ``` 59 | kubectl apply -f swarm-hostpath.yaml 60 | ``` 61 | 62 | The local path on the worker is /nvme/disk/clickhouse. /nvme/disk is a 63 | soft link on the worker that points to the NVMe SSD file system mount 64 | point. Swarms must use a mount point below this. 65 | 66 | ### Issues 67 | 68 | * During autoscaling the swarm node may get scheduled onto the worker 69 | before the eks-nvme-ssd-provisioner can format the disk and create 70 | the mount point. In this case the pod will not correctly mount the 71 | hostPath volume and will use the worker root storage instead. 72 | 73 | ## Swarm using NVMe with local persistent volumes (Experimental) 74 | 75 | This is based on the [Local Persistence Volume Static Provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) project. 76 | 77 | It attempts to create a PVs from the worker's local NVMe file system 78 | provisioned by the eks-nvme-ssd-provisioner. 79 | 80 | Local storage provisioning currently does not mesh 81 | well with cluster autoscaling. The issue is summarized in 82 | https://github.com/kubernetes/autoscaler/issues/1658#issuecomment-1036205889, 83 | which also provides a draft workaround to enable the Kubernetes cluster 84 | autoscaler to complete pod deployments when the pod depends on storage 85 | that is allocated locally on the VM once it scales up. 86 | 87 | ### Prerequisites 88 | 89 | * The daemonset pod spec must match the 90 | nodeSelector and have matching tolerations so that it can operate on 91 | worker nodes. 92 | 93 | * Storage class must use a fake provisioner name or scale-up will 94 | not start. See 95 | 96 | * The EKS nodegroup must have following EKS tag set to let the autoscaler 97 | infer the settings on worker nodes: 98 | `k8s.io/cluster-autoscaler/node-template/label/aws.amazon.com/eks-local-ssd=true` 99 | 100 | * All mount paths in the sig-storage-local-static-provisioner must 101 | match the eks-nvme-ssd-paths. The correct mount path is: `/pv-disks`. 102 | (The default path in generated yaml files is /dev/disk/kubernetes. This 103 | does not work.) 104 | 105 | The included [local-storage-eks-nvme-ssd.yaml file](./local-storage-eks-nvme-ssd.yaml) 106 | is adjusted to meet the above requirements other than the EKS tag on the swarm node 107 | group, which must be set manually for now. 108 | 109 | Install [Altinity Kubernetes Operator for ClickHouse](https://github.com/Altinity/clickhouse-operator). 110 | (Must be 0.24.4 or above to support CHK resource to manage ClickHouse Keeper.) 111 | 112 | ### Installation 113 | 114 | Install as follows: 115 | ``` 116 | kubectl apply -f local-storage-eks-nvme-ssd.yaml 117 | ``` 118 | 119 | Confirm that it is working by checking that a daemon appears on 120 | each new worker node. You should also see log messages when the daemon 121 | formats the file system, as shown by the following example. 122 | 123 | ``` 124 | $ kubectl logs local-static-provisioner-k9rqt|more 125 | I0330 05:10:19.357347 1 main.go:69] Loaded configuration: {StorageClass 126 | Config:map[nvme-ssd:{HostDir:/pv-disks MountDir:/pv-disks BlockCleanerCommand 127 | :[/scripts/quick_reset.sh] VolumeMode:Filesystem FsType: NamePattern:*}] Node 128 | LabelsForPV:[] UseAlphaAPI:false UseJobForCleaning:false MinResyncPeriod:{Dur 129 | ation:5m0s} UseNodeNameOnly:false LabelsForPV:map[] SetPVOwnerRef:false} 130 | I0330 05:10:19.357403 1 main.go:70] Ready to run... 131 | I0330 05:10:19.357464 1 common.go:444] Creating client using in-cluster 132 | config 133 | I0330 05:10:19.370946 1 main.go:95] Starting config watcher 134 | I0330 05:10:19.370964 1 main.go:98] Starting controller 135 | I0330 05:10:19.370971 1 main.go:102] Starting metrics server at :8080 136 | I0330 05:10:19.371060 1 controller.go:91] Initializing volume cache 137 | I0330 05:10:19.471437 1 controller.go:163] Controller started 138 | I0330 05:10:19.471697 1 discovery.go:423] Found new volume at host path 139 | "/pv-disks/fa6a9dd7-322b-456b-bc86-176c5dee2470" with capacity 921138413568, 140 | creating Local PV "local-pv-8312a141", required volumeMode "Filesystem" 141 | I0330 05:10:19.481257 1 cache.go:55] Added pv "local-pv-8312a141" to ca 142 | che 143 | I0330 05:10:19.481326 1 discovery.go:457] Created PV "local-pv-8312a141 144 | ``` 145 | 146 | ### Start swarm 147 | 148 | The swarm-local-storage.yaml is configured to use local storage PVs. 149 | 150 | ``` 151 | kubectl apply -f swarm-local-storage.yaml 152 | ``` 153 | 154 | For this to work you must currently scale up the node group manually to the number 155 | of requested nodes. 156 | 157 | ### Issues 158 | 159 | * Autoscaling does not work, because the cluster autoscaler will hang 160 | waiting for acknowledgement of PVs on new workers. 161 | 162 | * The local storage provisioner does not properly clean up PVs left 163 | behind when workers are deleted. This is probably a consequence of 164 | using a mock 165 | provisioner. 166 | 167 | * Behavior may also be flakey even when workers are 168 | preprovisioned. Scaling up one-by-one seems OK. 169 | -------------------------------------------------------------------------------- /kubernetes/README.md: -------------------------------------------------------------------------------- 1 | # Antalya Kubernetes Example 2 | 3 | This directory contains samples for querying a Parquet-based data lake 4 | lake using AWS EKS, AWS S3, and Project Antalya. 5 | 6 | ## Quickstart 7 | 8 | ### Prerequisites 9 | 10 | Install: 11 | * [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) 12 | * [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) 13 | * [terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) 14 | 15 | ### Start Kubernetes 16 | 17 | Cd to the terraform directory and follow the installation directions in the 18 | README.md file to set up a Kubernetes cluster on EKS. Here's the short form. 19 | 20 | ``` 21 | cd terraform 22 | terraform init 23 | terraform apply 24 | aws eks update-kubeconfig --name my-eks-cluster # Default cluster name 25 | ``` 26 | 27 | Create a namespace named antalya and make it the default. (You don't 28 | have to do this but the examples assume it.) 29 | 30 | ``` 31 | kubectl create ns antalya 32 | kubectl config set-context --current --namespace=antalya 33 | ``` 34 | 35 | ### Install the Altinity Operator for Kubernetes 36 | 37 | Install the latest production version of the [Altinity Kubernetes Operator 38 | for ClickHouse](https://github.com/Altinity/clickhouse-operator). 39 | 40 | ``` 41 | kubectl apply -f https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/deploy/operator/clickhouse-operator-install-bundle.yaml 42 | ``` 43 | 44 | ### Install an Iceberg REST catalog 45 | 46 | This step installs an Iceberg REST catalog using the 47 | Altinity [Ice Toolset](https://github.com/Altinity/ice). 48 | 49 | Follow instructions in the [ice directory README.md](ice/README.md). 50 | 51 | ### Install ClickHouse server with Antalya swarm cluster 52 | 53 | This step installs a ClickHouse "vector" server that applications can connect 54 | to, an Antalya swarm cluster, and a Keeper ensemble to allow the swarm servers 55 | to register themselves dynamically. 56 | 57 | #### Using plain manifest files 58 | 59 | Cd to the manifests directory and install the manifests in the default 60 | namespace. 61 | 62 | ``` 63 | cd manifest 64 | kubectl apply -f gp3-encrypted-fast-storage-class.yaml 65 | kubectl apply -f keeper.yaml 66 | kubectl apply -f swarm.yaml 67 | kubectl apply -f vector.yaml 68 | ``` 69 | 70 | #### Using helm 71 | 72 | The helm script is in the helm directory. It's under development. 73 | 74 | ## Running 75 | 76 | ### Querying Parquet files on AWS S3 and Apache Iceberg 77 | 78 | AWS kindly provides 79 | [AWS Public Block Data](https://registry.opendata.aws/aws-public-blockchain/), 80 | which we will use as example data for Parquet on S3. 81 | 82 | Start by logging into the vector server. 83 | ``` 84 | kubectl exec -it chi-vector-example-0-0-0 -- clickhouse-client 85 | ``` 86 | 87 | Try running a query using only the vector server. 88 | ``` 89 | SELECT date, sum(output_count) 90 | FROM s3('s3://aws-public-blockchain/v1.0/btc/transactions/**.parquet', NOSIGN) 91 | WHERE date >= '2025-01-01' GROUP BY date ORDER BY date ASC 92 | SETTINGS use_hive_partitioning = 1 93 | ``` 94 | 95 | This query sets the baseline for execution without assistance from the swarm. 96 | Depending on the date range you use it is likely to be slow. You can cancel 97 | using ^C. 98 | 99 | Next, let's try a query using the swarm. The object_storage_cluster 100 | setting points to the swarm cluster name. 101 | 102 | ``` 103 | SELECT date, sum(output_count) 104 | FROM s3('s3://aws-public-blockchain/v1.0/btc/transactions/**.parquet', NOSIGN) 105 | WHERE date >= '2025-02-01' GROUP BY date ORDER BY date ASC 106 | SETTINGS use_hive_partitioning = 1, object_storage_cluster = 'swarm'; 107 | ``` 108 | 109 | The next query shows results when caches are turned on. 110 | ``` 111 | SELECT date, sum(output_count) 112 | FROM s3('s3://aws-public-blockchain/v1.0/btc/transactions/**.parquet', NOSIGN) 113 | WHERE date >= '2025-02-01' GROUP BY date ORDER BY date ASC 114 | SETTINGS use_hive_partitioning = 1, object_storage_cluster = 'swarm', 115 | input_format_parquet_use_metadata_cache = 1, enable_filesystem_cache = 1; 116 | ``` 117 | 118 | Successive queries will complete faster as caches load. 119 | 120 | ### Improving performance by scaling up the swarm 121 | 122 | You can at any time increase the size of the swarm server by directly 123 | editing the swarm CHI resource, changing the number of shards to 8, 124 | and submitting the changes. (Example using manifest files.) 125 | 126 | ``` 127 | kubectl edit chi swarm 128 | ... 129 | podTemplate: replica 130 | volumeClaimTemplate: storage 131 | shardsCount: 4 <-- Change to 8 and save. 132 | templates: 133 | ... 134 | ``` 135 | 136 | Run the query again after scale-up completes. You should see the response 137 | time drop by roughly 50%. Try running it again. You should see a further drop 138 | as swarm caches pick up additional files. You can scale up further to see 139 | additional drops. This setup has been tested to 16 nodes. 140 | 141 | To scale down the swarm, just edit the shardsCount again and set it to 142 | a smaller number. 143 | 144 | Important note: You may see failed queries as the swarm scales down. This 145 | is [a known issue](https://github.com/Altinity/ClickHouse/issues/759) 146 | and will be corrected soon. 147 | 148 | ### Querying Parquet files in Iceberg 149 | 150 | You can load the public data set into Iceberg, which makes the queries 151 | much easier to construct. Here are examples of the same queries when 152 | the public data are available in Iceberg once you do the ice REST 153 | catalog installation. 154 | 155 | ``` 156 | SET allow_experimental_database_iceberg=true; 157 | 158 | -- Use this for Antalya 25.3 or above. 159 | CREATE DATABASE ice 160 | ENGINE = DataLakeCatalog('http://ice-rest-catalog:5000') 161 | SETTINGS catalog_type = 'rest', 162 | auth_header = 'Authorization: Bearer foo', 163 | warehouse = 's3://rhodges-ice-rest-catalog-demo}'; 164 | 165 | -- Use this for Antalya 25.2 or below. 166 | CREATE DATABASE ice 167 | ENGINE = Iceberg('https://rest-catalog.dev.altinity.cloud') 168 | SETTINGS catalog_type = 'rest', 169 | auth_header = 'Authorization: Bearer jj...2j', 170 | warehouse = 's3://aws...iceberg'; 171 | ``` 172 | 173 | Show the tables available in the database. 174 | 175 | ``` 176 | SHOW TABLES FROM ice 177 | 178 | ┌─name─────────────┐ 179 | 1. │ btc.transactions │ 180 | 2. │ nyc.taxis │ 181 | └──────────────────┘ 182 | ``` 183 | 184 | Try counting rows. This goes faster if you enable caching of Iceberg metadata. 185 | 186 | ``` 187 | SELECT count() 188 | FROM ice.`btc.transactions` 189 | SETTINGS use_hive_partitioning = 1, object_storage_cluster = 'swarm', 190 | input_format_parquet_use_metadata_cache = 1, enable_filesystem_cache = 1, 191 | use_iceberg_metadata_files_cache=1; 192 | ``` 193 | 194 | Now try the same query that we ran earlier directly against the public S3 195 | dataset files. Caches are not enabled. 196 | 197 | ``` 198 | SELECT date, sum(output_count) 199 | FROM ice.`btc.transactions` 200 | WHERE date >= '2025-02-01' GROUP BY date ORDER BY date ASC 201 | SETTINGS use_hive_partitioning = 1, object_storage_cluster = 'swarm'; 202 | ``` 203 | 204 | Try the same query with all caches enabled. It should be faster. 205 | 206 | ``` 207 | SELECT date, sum(output_count) 208 | FROM ice.`btc.transactions` 209 | WHERE date = '2025-02-01' GROUP BY date ORDER BY date ASC 210 | SETTINGS use_hive_partitioning = 1, object_storage_cluster = 'swarm', 211 | input_format_parquet_use_metadata_cache = 1, enable_filesystem_cache = 1, 212 | use_iceberg_metadata_files_cache = 1; 213 | ``` 214 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Antalya Docker Example 2 | 3 | This directory contains samples for construction of an Iceberg-based data 4 | lake using Docker Compose and Altinity Antalya. 5 | 6 | The docker compose structure and the Python scripts took early inspiration from 7 | [ClickHouse integration tests for Iceberg](https://github.com/ClickHouse/ClickHouse/tree/master/tests/integration/test_database_iceberg) but have deviated substantially since then. 8 | 9 | ## Quickstart 10 | 11 | Examples are for Ubuntu. Adjust commands for other distros. 12 | 13 | ### Install prerequisite software 14 | 15 | Install [Docker Desktop](https://docs.docker.com/engine/install/) and 16 | [Docker Compose](https://docs.docker.com/compose/install/). 17 | 18 | Install the Altinity ice catalog client. (Requires a JDK.) 19 | 20 | ``` 21 | sudo apt install openjdk-21-jdk 22 | curl -sSL https://github.com/altinity/ice/releases/download/v0.8.1/ice-0.8.1 \ 23 | -o ice && chmod a+x ice && sudo mv ice /usr/local/bin/ 24 | ``` 25 | 26 | ### Bring up the data lake 27 | 28 | ``` 29 | docker compose up -d 30 | ``` 31 | 32 | ### Load data 33 | 34 | Create a table by loading using the ice catalog client. This creates the 35 | table automatically from the schema in the parquet file. 36 | 37 | ``` 38 | ice insert nyc.taxis -p \ 39 | https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet 40 | ``` 41 | 42 | ### Compute aggregates on Parquet data 43 | 44 | Connect to the Antalya server container and start clickhouse-client. 45 | ``` 46 | docker exec -it vector clickhouse-client 47 | ``` 48 | 49 | Set up database pointing to Ice[berg] REST catalog. 50 | ``` 51 | SET allow_experimental_database_iceberg = 1; 52 | 53 | DROP DATABASE IF EXISTS ice; 54 | 55 | CREATE DATABASE ice ENGINE = DataLakeCatalog('http://ice-rest-catalog:5000') 56 | SETTINGS catalog_type = 'rest', 57 | auth_header = 'Authorization: Bearer foo', 58 | storage_endpoint = 'http://minio:9000', 59 | warehouse = 's3://warehouse'; 60 | ``` 61 | 62 | Query data on vector only, followed by vector plus swarm servers. 63 | ``` 64 | -- Run query only on initiator. 65 | SELECT 66 | toDate(tpep_pickup_datetime) AS date, 67 | avg(passenger_count) AS passengers, 68 | avg(fare_amount) AS fare 69 | FROM ice.`nyc.taxis` GROUP BY date ORDER BY date 70 | 71 | -- Delegate to swarm servers. 72 | SELECT 73 | toDate(tpep_pickup_datetime) AS date, 74 | avg(passenger_count) AS passengers, 75 | avg(fare_amount) AS fare 76 | FROM ice.`nyc.taxis` GROUP BY date ORDER BY date 77 | SETTINGS object_storage_cluster='swarm' 78 | ``` 79 | 80 | ### Bring down cluster and delete data 81 | 82 | ``` 83 | docker compose down 84 | sudo rm -rf data 85 | ``` 86 | 87 | ## Load data with Python and view with ClickHouse 88 | 89 | ### Enable Python 90 | 91 | Install Python virtual environment module for your python version. Example shown 92 | below for Python 3.12. 93 | 94 | ``` 95 | sudo apt install python3.12-venv 96 | ``` 97 | 98 | Create and invoke the venv, then install required modules. 99 | ``` 100 | python3.12 -m venv venv 101 | . ./venv/bin/activate 102 | pip install --upgrade pip 103 | pip install -r requirements.txt 104 | ``` 105 | 106 | ### Load and read data with pyiceberg library 107 | ``` 108 | python iceberg_setup.py 109 | python iceberg_read.py 110 | ``` 111 | 112 | ### Demonstrate Antalya queries against data from Python 113 | 114 | Connect to the Antalya server container and start clickhouse-client. 115 | ``` 116 | docker exec -it vector clickhouse-client 117 | ``` 118 | 119 | Query data on vector only, followed by vector plus swarm servers. 120 | ``` 121 | -- Run query only on initiator. 122 | SELECT * FROM ice.`iceberg.bids` 123 | 124 | -- Delegate to swarm servers. 125 | SELECT symbol, avg(bid) 126 | FROM ice.`iceberg.bids` GROUP BY symbol 127 | SETTINGS object_storage_cluster = 'swarm' 128 | ``` 129 | 130 | ## Using Spark with ClickHouse and Ice 131 | 132 | Connect to the spark-iceberg container command line. 133 | ``` 134 | docker exec -it spark-iceberg /bin/bash 135 | ``` 136 | 137 | Start the Spark scala shell. 138 | ``` 139 | spark-shell 140 | ``` 141 | 142 | Read data and prove you can change it as well by running the commands below. 143 | ``` 144 | spark.sql("SHOW NAMESPACES").show() 145 | spark.sql("SHOW TABLES FROM iceberg").show() 146 | spark.sql("SHOW CREATE TABLE iceberg.bids").show(truncate=false) 147 | spark.sql("SELECT * FROM iceberg.bids").show() 148 | spark.sql("DELETE FROM iceberg.bids WHERE bid < 198.23").show() 149 | spark.sql("SELECT * FROM iceberg.bids").show() 150 | ``` 151 | 152 | Try reading the table from ClickHouse. The deleted rows should be gone. 153 | 154 | ## Additional help and troubleshooting 155 | 156 | ### Logs 157 | 158 | Logs are in the data directory along with service data. 159 | 160 | ### Cleaning up 161 | 162 | This deletes *all* containers and volumes for a fresh start. Do not use it 163 | if you have other Docker applications running. 164 | ``` 165 | ./clean-all.sh -f 166 | ``` 167 | 168 | ### Find out where your query ran 169 | 170 | If you are curious to find out where your query was actually processed, 171 | you can find out easily. Take the query_id that clickhouse-client prints 172 | and run a query like the following. You'll see all query log records. 173 | 174 | ``` 175 | SELECT hostName() AS host, type, initial_query_id, is_initial_query, query 176 | FROM clusterAllReplicas('all', system.query_log) 177 | WHERE (type = 'QueryFinish') 178 | AND (initial_query_id = '8051eef1-e68b-491a-b63d-fac0c8d6ef27')\G 179 | ``` 180 | 181 | ### Setting up Iceberg databases 182 | 183 | These commands when the vector server comes up for the first time. 184 | 185 | ``` 186 | SET allow_experimental_database_iceberg = 1; 187 | 188 | DROP DATABASE IF EXISTS ice; 189 | 190 | CREATE DATABASE ice 191 | ENGINE = DataLakeCatalog('http://ice-rest-catalog:5000') 192 | SETTINGS catalog_type = 'rest', 193 | auth_header = 'Authorization: Bearer foo', 194 | storage_endpoint = 'http://minio:9000', 195 | warehouse = 's3://warehouse'; 196 | ``` 197 | 198 | ### Query Iceberg and local data together 199 | 200 | Create a local table and populate it with data from Iceberg, altering 201 | data to make it different. 202 | 203 | ``` 204 | CREATE DATABASE IF NOT EXISTS local 205 | ; 206 | CREATE TABLE local.bids AS datalake.`iceberg.bids` 207 | ENGINE = MergeTree 208 | PARTITION BY toDate(datetime) 209 | ORDER BY (symbol, datetime) 210 | SETTINGS allow_nullable_key = 1 211 | ; 212 | -- Pull some data into the local table, making it look different. 213 | INSERT INTO local.bids 214 | SELECT datetime + toIntervalDay(4), symbol, bid, ask 215 | FROM datalake.`iceberg.bids` 216 | ; 217 | SELECT * 218 | FROM local.bids 219 | UNION ALL 220 | SELECT * 221 | FROM datalake.`iceberg.bids` 222 | ; 223 | -- Create a merge table. 224 | CREATE TABLE all_bids AS local.bids 225 | ENGINE = Merge(REGEXP('local|datalake'), '.*bids') 226 | ; 227 | SELECT * FROM all_bids 228 | ; 229 | ``` 230 | 231 | ### Fetching values from Iceberg catalog using curl 232 | 233 | The Iceberg REST API is simple to query using curl. The documentation is 234 | effectively [the full REST spec in the Iceberg GitHub Repo](https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml). Meanwhile here 235 | are a few examples that you can try on this project. 236 | 237 | Find namespaces. 238 | ``` 239 | curl -H "Authorization: bearer foo" http://localhost:5000/v1/namespaces | jq -s 240 | ``` 241 | 242 | Find tables in namespace. 243 | ``` 244 | curl -H "Authorization: bearer foo" http://localhost:5000/v1/namespaces/iceberg/tables | jq -s 245 | ``` 246 | 247 | Find table spec in Iceberg. 248 | ``` 249 | curl -H "Authorization: bearer foo" http://localhost:5000/v1/namespaces/iceberg/tables/bids | jq -s 250 | ``` 251 | -------------------------------------------------------------------------------- /docker/tests/helpers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import base64 16 | import os 17 | import subprocess 18 | import sys 19 | import time 20 | 21 | import requests 22 | 23 | 24 | class DockerHelper: 25 | """Helper class containing functions to manage Docker Compose operations 26 | including log capture as well as other actions require to test the 27 | docker compose setup""" 28 | 29 | def __init__(self, docker_dir): 30 | """Initialize DockerHelper with the Docker Compose directory""" 31 | self.docker_dir = docker_dir 32 | self.started_services = False 33 | self._create_logs_directory() 34 | 35 | def _create_logs_directory(self): 36 | """Create the test_logs directory if it doesn't exist""" 37 | logs_dir = os.path.join(self.docker_dir, "test_logs") 38 | os.makedirs(logs_dir, exist_ok=True) 39 | 40 | def setup_services(self): 41 | """Start Docker Compose services or verify existing setup based on environment""" 42 | os.chdir(self.docker_dir) 43 | 44 | if os.getenv('SKIP_DOCKER_SETUP') == 'true': 45 | print("Using existing Docker Compose setup...") 46 | self._verify_services_running() 47 | else: 48 | print("Starting Docker Compose services...") 49 | self._start_docker_services() 50 | self.started_services = True 51 | 52 | def cleanup_services(self): 53 | """Capture logs and stop Docker Compose services if we started them""" 54 | # Always capture logs for debugging 55 | self.capture_container_logs() 56 | 57 | # Only stop services if we started them 58 | if self.started_services: 59 | print("Stopping Docker Compose services...") 60 | self._stop_docker_services() 61 | 62 | def _start_docker_services(self): 63 | """Start Docker Compose services""" 64 | try: 65 | subprocess.run( 66 | ["docker", "compose", "up", "-d"], 67 | capture_output=True, 68 | text=True, 69 | check=True, 70 | ) 71 | print("Docker Compose started successfully") 72 | 73 | # Wait a bit for services to be ready 74 | time.sleep(5) 75 | except subprocess.CalledProcessError as e: 76 | print(f"Failed to start Docker Compose: {e}") 77 | print(f"stdout: {e.stdout}") 78 | print(f"stderr: {e.stderr}") 79 | sys.exit(1) 80 | 81 | def _stop_docker_services(self): 82 | """Stop Docker Compose services""" 83 | try: 84 | subprocess.run( 85 | ["docker", "compose", "down"], 86 | capture_output=True, 87 | text=True, 88 | check=True, 89 | ) 90 | print("Docker Compose stopped successfully") 91 | except subprocess.CalledProcessError as e: 92 | print(f"Failed to stop Docker Compose: {e}") 93 | print(f"stdout: {e.stdout}") 94 | print(f"stderr: {e.stderr}") 95 | 96 | def _verify_services_running(self): 97 | """Verify that required Docker Compose services are running""" 98 | try: 99 | result = subprocess.run( 100 | ["docker", "compose", "ps", "--services", "--filter", "status=running"], 101 | capture_output=True, 102 | text=True, 103 | check=True, 104 | ) 105 | running_services = result.stdout.strip().split('\n') if result.stdout.strip() else [] 106 | if not running_services: 107 | print("Warning: No running Docker Compose services found") 108 | else: 109 | print(f"Found running services: {', '.join(running_services)}") 110 | except subprocess.CalledProcessError as e: 111 | print(f"Failed to verify services: {e}") 112 | print(f"stdout: {e.stdout}") 113 | print(f"stderr: {e.stderr}") 114 | 115 | def capture_container_logs(self): 116 | """Capture container logs and save them to test_logs directory""" 117 | print("Capturing container logs...") 118 | logs_dir = os.path.join(self.docker_dir, "test_logs") 119 | timestamp = time.strftime("%Y%m%d_%H%M%S") 120 | 121 | try: 122 | # Get list of services 123 | result = subprocess.run( 124 | ["docker", "compose", "config", "--services"], 125 | capture_output=True, 126 | text=True, 127 | check=True, 128 | ) 129 | services = result.stdout.strip().split('\n') if result.stdout.strip() else [] 130 | 131 | # Capture logs for each service 132 | for service in services: 133 | if service: # Skip empty lines 134 | try: 135 | log_result = subprocess.run( 136 | ["docker", "compose", "logs", "--no-color", service], 137 | capture_output=True, 138 | text=True, 139 | check=True, 140 | ) 141 | log_file = os.path.join(logs_dir, f"{service}_{timestamp}.log") 142 | with open(log_file, 'w') as f: 143 | f.write(f"=== Logs for service: {service} ===\n") 144 | f.write(f"=== Captured at: {time.strftime('%Y-%m-%d %H:%M:%S')} ===\n\n") 145 | f.write(log_result.stdout) 146 | print(f"Saved logs for {service} to {log_file}") 147 | except subprocess.CalledProcessError as e: 148 | print(f"Failed to capture logs for service {service}: {e}") 149 | 150 | # Also capture combined logs 151 | try: 152 | combined_result = subprocess.run( 153 | ["docker", "compose", "logs", "--no-color"], 154 | capture_output=True, 155 | text=True, 156 | check=True, 157 | ) 158 | combined_log_file = os.path.join(logs_dir, f"combined_{timestamp}.log") 159 | with open(combined_log_file, 'w') as f: 160 | f.write(f"=== Combined Docker Compose Logs ===\n") 161 | f.write(f"=== Captured at: {time.strftime('%Y-%m-%d %H:%M:%S')} ===\n\n") 162 | f.write(combined_result.stdout) 163 | print(f"Saved combined logs to {combined_log_file}") 164 | except subprocess.CalledProcessError as e: 165 | print(f"Failed to capture combined logs: {e}") 166 | 167 | except subprocess.CalledProcessError as e: 168 | print(f"Failed to get service list: {e}") 169 | print(f"stdout: {e.stdout}") 170 | print(f"stderr: {e.stderr}") 171 | 172 | 173 | def generate_basic_auth_header(username, password): 174 | """Generate a properly encoded basic authentication header""" 175 | credentials = f"{username}:{password}" 176 | encoded_credentials = base64.b64encode(credentials.encode("utf-8")).decode( 177 | "utf-8" 178 | ) 179 | return f"Basic {encoded_credentials}" 180 | 181 | 182 | def http_get_helper(test_case, url, timeout=10, expected_status_code=200, auth_header=None): 183 | """Helper function to perform HTTP GET request with error handling""" 184 | try: 185 | headers = {} 186 | if auth_header: 187 | headers["Authorization"] = auth_header 188 | 189 | response = requests.get(url, timeout=timeout, headers=headers) 190 | # Check if status code matches expected. 191 | if response.status_code == expected_status_code: 192 | print(f"HTTP GET to {url} successful: {response.status_code}") 193 | return response 194 | else: 195 | test_case.fail( 196 | f"Expected status {expected_status_code}, got {response.status_code}" 197 | ) 198 | except requests.exceptions.RequestException as e: 199 | print(f"HTTP request failed for URL: {url}") 200 | print(f"Exception: {e}") 201 | test_case.fail(f"HTTP request failed: {e}") 202 | 203 | 204 | def run_python_script_helper(test_case, script_name): 205 | """Helper function to run a Python script from the parent directory""" 206 | try: 207 | # Get the parent directory (docker) where the Python scripts should be 208 | test_dir = os.path.dirname(os.path.abspath(__file__)) 209 | docker_dir = os.path.dirname(test_dir) 210 | script_path = os.path.join(docker_dir, script_name) 211 | 212 | # Run the script and check for success 213 | result = subprocess.run( 214 | ["python", script_path], 215 | capture_output=True, 216 | text=True, 217 | check=True, 218 | cwd=docker_dir 219 | ) 220 | print(f"{script_name} executed successfully") 221 | print(f"stdout: {result.stdout}") 222 | return result 223 | except subprocess.CalledProcessError as e: 224 | test_case.fail(f"{script_name} failed with exit code {e.returncode}\n" 225 | f"stdout: {e.stdout}\nstderr: {e.stderr}") 226 | except FileNotFoundError: 227 | test_case.fail(f"{script_name} not found in parent directory") 228 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /docs/reference.md: -------------------------------------------------------------------------------- 1 | # Command and Configuration Reference 2 | 3 | ## SQL Syntax Guide 4 | 5 | This section shows how to use the Iceberg database engine, table engine, and 6 | table function. 7 | 8 | ### Iceberg Database Engine 9 | 10 | The Iceberg database engine connects ClickHouse to an Iceberg REST catalog. The 11 | tables listed in the REST catalog show up as database. The Iceberg REST catalog 12 | must already exist. Here is an example of the syntax. Note that you must enable 13 | Iceberg database support with the allow_experimental_database_iceberg. This can 14 | also be placed in a user profile to enable it by default. 15 | 16 | ``` 17 | SET allow_experimental_database_iceberg=true; 18 | 19 | CREATE DATABASE datalake 20 | ENGINE = Iceberg('http://rest:8181/v1', 'minio', 'minio123') 21 | SETTINGS catalog_type = 'rest', 22 | storage_endpoint = 'http://minio:9000/warehouse', 23 | warehouse = 'iceberg'; 24 | ``` 25 | 26 | The Iceberg database engine takes three arguments: 27 | 28 | * url - Path to Iceberg READ catalog endpoint 29 | * user - Object storage user 30 | * password - Object storage password 31 | 32 | The following settings are supported. 33 | 34 | * auth_header - Authorization header of format 'Authorization: ' 35 | * auth_scope - Authorization scope for client credentials or token exchange 36 | * oauth_server_uri - OAuth server URI 37 | * vended_credentials - Use vended credentials (storage credentials) from catalog 38 | * warehouse - Warehouse name inside the catalog 39 | * storage_endpoint - Object storage endpoint 40 | 41 | ### Iceberg Table Engine 42 | 43 | Will be documented later. 44 | 45 | ### Iceberg Table Function 46 | 47 | The [Iceberg table function](https://clickhouse.com/docs/en/sql-reference/table-functions/iceberg) 48 | selects from an Iceberg table. It uses the path of the table in object 49 | storage to locate table metadata. Here is an example of the syntax. 50 | 51 | ``` 52 | SELECT count() 53 | FROM iceberg('http://minio:9000/warehouse/data') 54 | ``` 55 | 56 | You can dispatch queries to the swarm as follows: 57 | 58 | ``` 59 | SELECT count() 60 | FROM iceberg('http://minio:9000/warehouse/data') 61 | SETTINGS object_storage_cluster = 'swarm' 62 | ``` 63 | 64 | The iceberg() function is an alias for icebergS3(). See the upstream docs for more information. 65 | 66 | It's important to note that the iceberg() table function expects to see data 67 | and metadata directores after the URL provided as an argument. In other words, 68 | the Iceberg table must be arranged in object storage as follows: 69 | 70 | * http://minio:9000/warehouse/data/metadata - Contains Iceberg metadata files for the table 71 | * http://minio:9000/warehouse/data/data - Contains Iceberg data files for the table 72 | 73 | If the files are not laid out as shown above the iceberg() table function 74 | may not be able to read data. 75 | 76 | ## Swarm Clusters 77 | 78 | Swarm clusters are clusters of stateless ClickHouse servers that may be used for parallel 79 | query on S3 files as well as Iceberg tables (which are just collections of S3 files). 80 | 81 | ### Using Swarm Clusters to speed up query 82 | 83 | Swarm clusters can accelerate queries that use any of the following functions. 84 | 85 | * s3() function 86 | * s3Cluster() function -- Specify as function argument 87 | * iceberg() function 88 | * icebergS3Cluster() function -- Specify as function argument 89 | * Iceberg table engine, including tables made available via using the Iceberg database engine 90 | 91 | To delegate subqueries to a swarm cluster, add the object_storage_cluster 92 | setting as shown below with the swarm cluster name. You can also set 93 | the value in a user profile, which will ensure that the setting applies by default 94 | to all queries for that user. 95 | 96 | Here's an example of a query on Parquet files using Hive partitioning. 97 | 98 | ``` 99 | SELECT hostName() AS host, count() 100 | FROM s3('http://minio:9000/warehouse/data/data/**/**.parquet') 101 | GROUP BY host 102 | SETTINGS use_hive_partitioning=1, object_storage_cluster='swarm' 103 | ``` 104 | 105 | Here is an example of querying the same data via Iceberg using the swarm 106 | cluster. 107 | 108 | ``` 109 | SELECT count() 110 | FROM datalake.`iceberg.bids` 111 | SETTINGS object_storage_cluster = 'swarm' 112 | ``` 113 | 114 | Here's an example of using the swarm cluster with the icebergS3Cluster() 115 | function. 116 | 117 | ``` 118 | SELECT hostName() AS host, count() 119 | FROM icebergS3Cluster('swarm', 'http://minio:9000/warehouse/data') 120 | GROUP BY host 121 | ``` 122 | 123 | ### Relevant settings for swarm clusters 124 | 125 | The following list shows the main query settings that affect swarm 126 | cluster processing. 127 | 128 | | Setting Name | Description | Value | 129 | |--------------|-------------|-------| 130 | | `enable_filesystem_cache` | Use filesystem cache for S3 blocks | 0 or 1 | 131 | | `input_format_parquet_use_metadata_cache` | Cache Parquet file metadata | 0 or 1 | 132 | | `input_format_parquet_metadata_cache_max_size` | Parquet metadata cache size (defaults to 500MiB) | Integer | 133 | | `object_storage_cluster` | Swarm cluster name | String | 134 | | `object_storage_max_nodes` | Number of swarm nodes to use (defaults to all nodes) | Integer | 135 | | `use_hive_partitioning` | Files follow Hive partitioning | 0 or 1 | 136 | | `use_iceberg_metadata_files_cache` | Cache parsed Iceberg metadata files in memory | 0 or 1 | 137 | | `use_iceberg_partition_pruning` | Prune files based on Iceberg data | 0 or 1 | 138 | 139 | ### Configuring swarm cluster autodiscovery 140 | 141 | Cluster-autodiscovery uses [Zoo]Keeper as a registry for swarm cluster 142 | members. Swarm cluster servers register themselves on a specific path 143 | at start-up time to join the cluster. Other servers can read the path 144 | find members of the swarm cluster. 145 | 146 | To use auto-discovery, you must enable Keeper by adding a `` 147 | tag similar to the following example. This must be done for all servers 148 | including swarm servers as well as ClickHouse servers that invoke them. 149 | 150 | ``` 151 | 152 | 153 | 154 | keeper 155 | 9181 156 | 157 | 158 | 159 | ``` 160 | 161 | You must also enable automatic cluster discovery. 162 | ``` 163 | 1 164 | ``` 165 | 166 | #### Using a single Keeper ensemble 167 | 168 | When using a single Keeper for all servers, add the following remote server 169 | definition to each swarm server configuration. This provides a path on which 170 | the server will register. 171 | 172 | ``` 173 | 174 | 175 | 176 | 177 | /clickhouse/discovery/swarm 178 | secret_key 179 | 180 | 181 | 182 | ``` 183 | 184 | Add the following remote server definition to each server that _reads_ the 185 | swarm server list using remote discovery. Note the `` tag, which 186 | must be set to prevent non-swarm servers from joining th cluster. 187 | 188 | ``` 189 | 190 | 191 | 192 | 193 | /clickhouse/discovery/swarm 194 | secret_key 195 | 196 | true 197 | 198 | 199 | 200 | ``` 201 | 202 | #### Using multiple keeper ensembles 203 | 204 | It's common to use separate keeper ensembles to manage intra-cluster 205 | replication and swarm cluster discovery. In this case you can enable 206 | an auxiliary keeper that handles only auto-discovery. Here is the 207 | configuration for such a Keeper ensemble. ClickHouse will 208 | use this Keeper ensemble for auto-discovery. 209 | 210 | ``` 211 | 212 | 213 | 214 | 215 | 216 | keeper 217 | 9181 218 | 219 | 220 | 221 | 222 | ``` 223 | 224 | This is in addition to the settings described in previous sections, 225 | which remain the same. 226 | 227 | ## Configuring Caches 228 | 229 | Caches make a major difference in the performance of ClickHouse queries. This 230 | section describes how to configure them in a swarm cluster. 231 | 232 | ### Iceberg Metadata Cache 233 | 234 | The Iceberg metadata cache keeps parsed table definitions in memory. It is 235 | enabled using the `use_iceberg_metadata_files_cache setting`, as shown in the 236 | following example: 237 | 238 | ``` 239 | SELECT count() 240 | FROM ice.`aws-public-blockchain.btc` 241 | SETTINGS object_storage_cluster = 'swarm', 242 | use_iceberg_metadata_files_cache = 0; 243 | ``` 244 | 245 | Reading and parsing Iceberg metadata files (including metadata.json, 246 | manifest list, and manifest files) is slow. Enabling this setting can 247 | speed up query planning significantly. 248 | 249 | ### Parquet Metadata Cache 250 | 251 | The Parquet metadata cache keeps metadata from individual Parquets in memory, 252 | including column metadata, min/max statistics, and Bloom filter indexes. 253 | Swarm nodes use the metadata to avoid fetching unnecessary blocks from 254 | object storage. If no blocks are needed the swarm node skips the file entirely. 255 | 256 | The following example shows how to enable Parquet metadata caching. 257 | ``` 258 | SELECT count() 259 | FROM ice.`aws-public-blockchain.btc` 260 | SETTINGS object_storage_cluster = 'swarm', 261 | input_format_parquet_use_metadata_cache = 1; 262 | ``` 263 | 264 | The server setting `input_format_parquet_metadata_cache_max_size` controls the 265 | size of the cache. It currently defaults to 500MiB. 266 | 267 | ### S3 Filesystem Cache 268 | 269 | This cache stores blocks read from object storage on local disk. It offers 270 | a considerable speed advantage, especially when blocks are in storage. The 271 | S3 filesystem cache requires special configuration each swarm host. 272 | 273 | #### Define the cache 274 | 275 | Add a definition like the following to /etc/clickhouse/filesystem_cache.xml 276 | to set up a filesystem cache. 277 | 278 | ``` 279 | spec: 280 | configuration: 281 | files: 282 | config.d/filesystem_cache.xml: | 283 | 284 | 285 | 286 | /var/lib/clickhouse/s3_parquet_cache 287 | 50Gi 288 | 289 | 290 | 291 | ``` 292 | 293 | #### Enable cache use in queries 294 | 295 | The following settings control use of the filesystem cache. 296 | 297 | * enable_filesystem_cache - Enable filesystem cache (1=enabled) 298 | * enable_filesystem_cache_log - Enable logging of cache operations (1=enabled) 299 | * filesystem_cache_name - Name of the cache to use (must be specified) 300 | 301 | You can enable the settings on a query as follows: 302 | 303 | ``` 304 | SELECT date, sum(output_count) 305 | FROM s3('s3://aws-public-blockchain/v1.0/btc/transactions/**.parquet', NOSIGN) 306 | WHERE date >= '2025-01-01' GROUP BY date ORDER BY date ASC 307 | SETTINGS use_hive_partitioning = 1, object_storage_cluster = 'swarm', 308 | enable_filesystem_cache = 1, filesystem_cache_name = 's3_parquet_cache' 309 | ``` 310 | 311 | You can also set cache values in user profiles as shown by the following 312 | settings in Altinity operator format: 313 | 314 | ``` 315 | spec: 316 | configuration: 317 | profiles: 318 | use_cache/enable_filesystem_cache: 1 319 | use_cache/enable_filesystem_cache_log: 1 320 | use_cache/filesystem_cache_name: "s3_parquet_cache" 321 | ``` 322 | 323 | #### Clear cache 324 | 325 | Issue the following command on any swarm server. (It does not work from 326 | other clusters.) 327 | 328 | ``` 329 | SYSTEM DROP FILESYSTEM CACHE ON CLUSTER 'swarm' 330 | ``` 331 | 332 | #### Find out how the cache is doing. 333 | 334 | Get statistics on file system caches across the swarm. 335 | 336 | ``` 337 | SELECT hostName() host, cache_name, count() AS segments, sum(size) AS size, 338 | min(cache_hits) AS min_hits, avg(cache_hits) AS avg_hits, 339 | max(cache_hits) AS max_hits 340 | FROM clusterAllReplicas('swarm', system.filesystem_cache) 341 | GROUP BY host, cache_name 342 | ORDER BY host, cache_name ASC 343 | FORMAT Vertical 344 | ``` 345 | 346 | Find out how many S3 calls an individual ClickHouse is making. When caching 347 | is working properly you should see the values remain the same between 348 | successive queries. 349 | 350 | ``` 351 | SELECT name, value 352 | FROM clusterAllReplicas('swarm', system.events) 353 | WHERE event ILIKE '%s3%' 354 | ORDER BY 1 355 | ``` 356 | 357 | To see S3 stats across all servers, use the following. 358 | ``` 359 | SELECT hostName() host, name, sum(value) AS value 360 | FROM clusterAllReplicas('all', system.events) 361 | WHERE event ILIKE '%s3%' 362 | GROUP BY 1, 2 ORDER BY 1, 2 363 | ``` 364 | 365 | To see S3 stats for a single query spread across multiple hosts, issue the following request. 366 | ``` 367 | SELECT hostName() host, k, v 368 | FROM clusterAllReplicas('all', system.query_log) 369 | ARRAY JOIN ProfileEvents.keys AS k, ProfileEvents.values AS v 370 | WHERE initial_query_id = '5737ecca-c066-42f8-9cd1-a910a3d1e0b4' AND type = 2 371 | AND k ilike '%S3%' 372 | ORDER BY host, k 373 | ``` 374 | 375 | ### S3 List Objects Cache 376 | 377 | Listing files in object storage using the S3 ListObjectsV2 call is 378 | expensive. The S3 List Objects Cache avoids repeated calls and can 379 | cut down significant time during query planning. You can enable it 380 | using the `use_object_storage_list_objects_cache` setting as shown below. 381 | 382 | ``` 383 | SELECT date, count() 384 | FROM s3('s3://aws-public-blockchain/v1.0/btc/transactions/*/*.parquet', NOSIGN) 385 | WHERE (date >= '2025-01-01') AND (date <= '2025-01-31') 386 | GROUP BY date 387 | ORDER BY date ASC 388 | SETTINGS use_hive_partitioning = 1, use_object_storage_list_objects_cache = 1 389 | ``` 390 | 391 | The setting can speed up performance enormously but has a number of limitations: 392 | 393 | * It does not speed up Iceberg queries, since Iceberg metadata provides lists 394 | of files. 395 | * It is best for datasets that are largely read-only. It may cause queries to miss 396 | newer files, if they arrive while the cache is active. 397 | --------------------------------------------------------------------------------