├── .github ├── deps.edn └── workflows │ ├── dockerhub.yaml │ ├── release.yaml │ └── tests.yaml ├── .gitignore ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── bin └── build_docker_image.sh ├── deps.edn ├── docker-compose.yml ├── resources └── metabase-plugin.yaml ├── scripts └── exclude_tests.diff ├── src └── metabase │ └── driver │ └── materialize.clj └── test └── metabase ├── driver └── materialize_test.clj └── test └── data └── materialize.clj /.github/deps.edn: -------------------------------------------------------------------------------- 1 | {:aliases 2 | {:user/materialize 3 | {:extra-paths ["PWD/modules/drivers/materialize/test"] 4 | :extra-deps {metabase/materialize {:local/root "PWD/modules/drivers/materialize"}}}}} 5 | -------------------------------------------------------------------------------- /.github/workflows/dockerhub.yaml: -------------------------------------------------------------------------------- 1 | name: Build and Publish Docker Image 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | workflow_dispatch: 8 | inputs: 9 | version: 10 | description: "Version to build and publish" 11 | required: true 12 | 13 | jobs: 14 | dockerhub: 15 | runs-on: ubuntu-latest 16 | permissions: 17 | contents: write 18 | steps: 19 | - name: Checkout Metabase Repo 20 | uses: actions/checkout@v4 21 | with: 22 | repository: metabase/metabase 23 | ref: release-x.54.x 24 | 25 | - name: Checkout Driver Repo 26 | uses: actions/checkout@v4 27 | with: 28 | path: modules/drivers/materialize 29 | 30 | - name: Set up JDK 21 31 | uses: actions/setup-java@v2 32 | with: 33 | distribution: temurin 34 | java-version: 21 35 | 36 | - name: Install Clojure CLI 37 | run: | 38 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1262.sh && 39 | sudo bash ./linux-install-1.11.1.1262.sh 40 | 41 | - name: Setup Node 42 | uses: actions/setup-node@v2 43 | with: 44 | node-version: "22" 45 | cache: "yarn" 46 | 47 | - name: Get M2 cache 48 | uses: actions/cache@v4 49 | with: 50 | path: | 51 | ~/.m2 52 | ~/.gitlibs 53 | key: ${{ runner.os }}-materialize-${{ hashFiles('**/deps.edn') }} 54 | 55 | - name: Prepare stuff for pulses 56 | run: yarn build-static-viz 57 | 58 | - name: Build Materialize driver 59 | run: | 60 | echo "{:deps {metabase/materialize {:local/root \"materialize\" }}}" > modules/drivers/deps.edn 61 | bin/build-driver.sh materialize 62 | mkdir -p modules/drivers/materialize/.build 63 | cp resources/modules/materialize.metabase-driver.jar modules/drivers/materialize/.build/materialize-driver.jar 64 | 65 | - name: Use QEMU for multi-platform images 66 | uses: docker/setup-qemu-action@v2 67 | 68 | - name: Use buildx for multi-platform images 69 | uses: docker/setup-buildx-action@v2 70 | 71 | - name: Log in to Docker Hub 72 | uses: docker/login-action@v2 73 | with: 74 | username: materializebot 75 | password: ${{ secrets.DOCKER_HUB_MATERIALIZEBOT_API_KEY }} 76 | 77 | - name: Extract metadata (tags, labels) for Docker 78 | id: meta 79 | uses: docker/metadata-action@v4 80 | with: 81 | images: materialize/metabase 82 | tags: | 83 | type=semver,pattern={{version}} 84 | 85 | - name: Build and push Docker image 86 | uses: docker/build-push-action@v4 87 | with: 88 | context: modules/drivers/materialize 89 | platforms: linux/amd64,linux/arm64 90 | push: true 91 | tags: ${{ steps.meta.outputs.tags }} 92 | labels: ${{ steps.meta.outputs.labels }} 93 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - "v*" 6 | 7 | jobs: 8 | release: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | contents: write 12 | steps: 13 | - name: Checkout Metabase Repo 14 | uses: actions/checkout@v4 15 | with: 16 | repository: metabase/metabase 17 | ref: release-x.54.x 18 | 19 | - name: Checkout Driver Repo 20 | uses: actions/checkout@v4 21 | with: 22 | path: modules/drivers/materialize 23 | 24 | - name: Set up JDK 21 25 | uses: actions/setup-java@v2 26 | with: 27 | distribution: temurin 28 | java-version: 21 29 | 30 | - name: Install Clojure CLI 31 | run: | 32 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1262.sh && 33 | sudo bash ./linux-install-1.11.1.1262.sh 34 | 35 | - name: Setup Node 36 | uses: actions/setup-node@v2 37 | with: 38 | node-version: "22" 39 | cache: "yarn" 40 | 41 | - name: Get M2 cache 42 | uses: actions/cache@v4 43 | with: 44 | path: | 45 | ~/.m2 46 | ~/.gitlibs 47 | key: ${{ runner.os }}-materialize-${{ hashFiles('**/deps.edn') }} 48 | 49 | - name: Prepare stuff for pulses 50 | run: yarn build-static-viz 51 | 52 | - name: Build Materialize driver 53 | run: | 54 | echo "{:deps {metabase/materialize {:local/root \"materialize\" }}}" > modules/drivers/deps.edn 55 | bin/build-driver.sh materialize 56 | ls -lah resources/modules 57 | 58 | - name: Install GitHub CLI 59 | run: | 60 | sudo apt update -y 61 | sudo apt install gh -y 62 | 63 | - name: Setup GitHub CLI 64 | working-directory: modules/drivers/materialize 65 | run: | 66 | gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}" 67 | 68 | - name: Create Release 69 | working-directory: modules/drivers/materialize 70 | run: | 71 | gh release create ${{ github.ref }} ../../../resources/modules/materialize.metabase-driver.jar -t "Release ${{ github.ref }}" -n "Release of the Materialize driver for Metabase" 72 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | branches: 7 | - master 8 | paths-ignore: 9 | - "**.md" 10 | schedule: 11 | - cron: 0 11 * * 0 12 | 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout Metabase Repo 18 | uses: actions/checkout@v4 19 | with: 20 | repository: metabase/metabase 21 | ref: release-x.54.x 22 | 23 | - name: Checkout Driver Repo 24 | uses: actions/checkout@v4 25 | with: 26 | path: modules/drivers/materialize 27 | 28 | - name: Set up JDK 21 29 | uses: actions/setup-java@v2 30 | with: 31 | distribution: temurin 32 | java-version: 21 33 | 34 | - name: Add Materialize TLS instance to /etc/hosts 35 | run: | 36 | sudo echo "127.0.0.1 materialize" | sudo tee -a /etc/hosts 37 | 38 | - name: Start Materialize in Docker 39 | uses: hoverkraft-tech/compose-action@v2.0.1 40 | with: 41 | compose-file: "modules/drivers/materialize/docker-compose.yml" 42 | services: | 43 | materialize 44 | 45 | # Apply the scripts/exclude_tests.diff patch to exclude tests that are not relevant to Materialize 46 | - name: Apply exclude_tests.diff 47 | run: | 48 | git apply modules/drivers/materialize/scripts/exclude_tests.diff 49 | 50 | - name: Install Clojure CLI 51 | run: | 52 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1262.sh && 53 | sudo bash ./linux-install-1.11.1.1262.sh 54 | 55 | - name: Setup Node 56 | uses: actions/setup-node@v2 57 | with: 58 | node-version: "22" 59 | cache: "yarn" 60 | 61 | - name: Get M2 cache 62 | uses: actions/cache@v4 63 | with: 64 | path: | 65 | ~/.m2 66 | ~/.gitlibs 67 | key: ${{ runner.os }}-materialize-${{ hashFiles('**/deps.edn') }} 68 | 69 | - name: Prepare stuff for pulses 70 | run: yarn build-static-viz 71 | 72 | # Use custom deps.edn containing "user/materialize" alias to include driver sources 73 | - name: Run tests 74 | run: | 75 | mkdir -p /home/runner/.config/clojure 76 | cat modules/drivers/materialize/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn 77 | 78 | # Retry tests up to 2 times as the Metabase test data sometimes fails to load on the first try 79 | ATTEMPTS=0 80 | MAX_RETRIES=2 81 | 82 | until [ $ATTEMPTS -ge $MAX_RETRIES ] 83 | do 84 | echo "Attempt $(($ATTEMPTS + 1)) of $MAX_RETRIES..." 85 | DRIVERS=materialize clojure -X:dev:drivers:drivers-dev:test:user/materialize && break 86 | ATTEMPTS=$(($ATTEMPTS + 1)) 87 | echo "Tests failed. Retrying in 10 seconds..." 88 | sleep 10 89 | done 90 | 91 | if [ $ATTEMPTS -eq $MAX_RETRIES ]; then 92 | echo "Tests failed after $MAX_RETRIES attempts." 93 | exit 1 94 | fi 95 | 96 | - name: Build Materialize driver 97 | run: | 98 | echo "{:deps {metabase/materialize {:local/root \"materialize\" }}}" > modules/drivers/deps.edn 99 | bin/build-driver.sh materialize 100 | ls -lah resources/modules 101 | 102 | - name: Archive driver JAR 103 | uses: actions/upload-artifact@v4 104 | with: 105 | name: materialize.metabase-driver.jar 106 | path: resources/modules/materialize.metabase-driver.jar 107 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .clj-kondo/ 3 | .lsp/ 4 | .cpcache/ 5 | .build 6 | .DS_Store 7 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Getting started 2 | 3 | * Please report any issues you encounter during operations. 4 | * Feel free to create a pull request, preferably with a test or five. 5 | 6 | ## Setting up a development environment 7 | 8 | ### Requirements 9 | 10 | * Clojure 1.11+ 11 | * OpenJDK 21 12 | * Node.js 22.x 13 | * Yarn 14 | 15 | For testing: [Docker Compose](https://docs.docker.com/compose/install/) 16 | 17 | Please refer to the extensive documentation available on the Metabase website: [Guide to writing a Metabase driver](https://www.metabase.com/docs/latest/developers-guide/drivers/start.html) 18 | 19 | Materialize driver's code should be inside the main Metabase repository checkout in `modules/drivers/materialize` directory. 20 | 21 | The easiest way to set up a development environment is as follows (mostly the same as in the [CI](https://github.com/MaterializeInc/metabase-materialize-driver/blob/master/.github/workflows/tests.yml)): 22 | 23 | * Clone Metabase and Materialize driver repositories 24 | ```bash 25 | git clone https://github.com/metabase/metabase.git 26 | cd metabase 27 | git checkout release-x.54.x 28 | git clone https://github.com/MaterializeInc/metabase-materialize-driver.git modules/drivers/materialize 29 | ``` 30 | 31 | * Create custom Clojure profiles, you can get it using the following command: 32 | 33 | ```bash 34 | cat modules/drivers/materialize/.github/deps.edn | sed -e "s|PWD|$PWD|g" | tr -d '\n' 35 | ``` 36 | 37 | Modifying `~/.clojure/deps.edn` will create two useful profiles: `user/materialize` that adds driver's sources to the classpath, and `user/test` that includes all the Metabase tests that are guaranteed to work with the driver. 38 | 39 | * Install the Metabase dependencies: 40 | 41 | ```bash 42 | clojure -X:deps:drivers prep 43 | ``` 44 | 45 | * Build the frontend: 46 | 47 | ```bash 48 | yarn && yarn build-static-viz 49 | ``` 50 | 51 | * Add `/etc/hosts` entry 52 | 53 | Required for TLS tests. 54 | 55 | ```bash 56 | sudo -- sh -c "echo 127.0.0.1 materialize >> /etc/hosts" 57 | ``` 58 | 59 | * Start Materialize as a Docker container 60 | 61 | ```bash 62 | docker compose -f modules/drivers/materialize/docker-compose.yml up -d materialize 63 | ``` 64 | 65 | * To use a specific Java version, you can set the `JAVA_HOME` environment variable: 66 | 67 | ```bash 68 | export JAVA_HOME=$(/usr/libexec/java_home -v 21) ; export PATH=$JAVA_HOME/bin:$PATH 69 | ``` 70 | 71 | Now, you should be able to run the tests: 72 | 73 | ```bash 74 | mz_deps=$(cat modules/drivers/materialize/.github/deps.edn | sed -e "s|PWD|$PWD|g" | tr -d '\n') 75 | DRIVERS=materialize clojure -Sdeps ${mz_deps} -X:dev:drivers:drivers-dev:test:user/materialize 76 | ``` 77 | 78 | you can see that we have our profiles `:user/materialize:user/test` added to the command above, and with `DRIVERS=materialize` we instruct Metabase to run the tests only for Materialize. 79 | 80 | > **Note** Omitting `DRIVERS` will run the tests for all the built-in database drivers. 81 | 82 | If you want to run tests for only a specific test: 83 | 84 | ```bash 85 | mz_deps=$(cat modules/drivers/materialize/.github/deps.edn | sed -e "s|PWD|$PWD|g" | tr -d '\n') 86 | DRIVERS=materialize clojure -Sdeps ${mz_deps} -X:dev:drivers:drivers-dev:test:user/materialize :only metabase.query-processor.middleware.parameters.mbql-test 87 | ``` 88 | 89 | ## Excluding tests 90 | 91 | Some tests are not applicable to Materialize, and you can exclude them by adding the following to the test command: 92 | 93 | ```bash 94 | git apply modules/drivers/materialize/scripts/exclude_tests.diff 95 | ``` 96 | 97 | The diff file contains the list of tests that are excluded from the Materialize driver. This often needs to be updated as new tests are added to new Metabase versions. 98 | 99 | ## Building a jar 100 | 101 | You need to add an entry for Materialize in `modules/drivers/deps.edn` 102 | 103 | ```clj 104 | {:deps 105 | {... 106 | metabase/materialize {:local/root "materialize"} 107 | ...}} 108 | ``` 109 | 110 | or just run this from the root Metabase directory, overwriting the entire file: 111 | 112 | ```bash 113 | echo "{:deps {metabase/materialize {:local/root \"materialize\" }}}" > modules/drivers/deps.edn 114 | ``` 115 | 116 | Now, you should be able to build the final jar: 117 | 118 | ```bash 119 | bin/build-driver.sh materialize 120 | ``` 121 | 122 | As the result, `resources/modules/materialize.metabase-driver.jar` should be created. You can copy it to the Metabase `plugins` directory and start Metabase. 123 | 124 | For smoke testing, there is a Metabase with the link to the driver available as a Docker container: 125 | 126 | ```bash 127 | docker compose -f modules/drivers/materialize/docker-compose.yml up -d metabase 128 | ``` 129 | 130 | It should pick up the driver jar as a volume. 131 | 132 | ## Cutting a release 133 | 134 | Once the driver is ready to be released and tests are passing, you need to create a tag in the Metabase repository: 135 | 136 | ```bash 137 | git tag -a vX.Y.Z -m vX.Y.Z 138 | git push origin vX.Y.Z 139 | ``` 140 | 141 | Once the tag is pushed, the CI will build the driver and create a release including the jar file. You can manually edit the release description to include any additional release notes. 142 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG METABASE_VERSION=latest 2 | FROM metabase/metabase:${METABASE_VERSION} 3 | 4 | ADD .build/materialize-driver.jar /plugins/ 5 | RUN chmod 744 /plugins/materialize-driver.jar 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 1997, PostgreSQL Global Development Group 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 14 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 17 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 19 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 20 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 21 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 22 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 23 | POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Materialize driver for Metabase 2 | 3 | [![Slack Badge](https://img.shields.io/badge/Join%20us%20on%20Slack!-blueviolet?style=flat&logo=slack&link=https://materialize.com/s/chat)](https://materialize.com/s/chat) 4 | 5 | The `metabase-materialize-driver` lets 6 | [Metabase](https://github.com/metabase/metabase) connect to an instance of 7 | [Materialize](https://github.com/MaterializeInc/materialize). 8 | 9 | > [!NOTE] 10 | > With improvements to Materialize's PostgreSQL compatibility, you can now connect to Materialize directly using Metabase's default PostgreSQL driver. 11 | > See the guide here: [Using Materialize with Metabase](https://materialize.com/docs/serve-results/metabase/). 12 | 13 | ![Choose Materialize from database dropdown](https://github-production-user-asset-6210df.s3.amazonaws.com/21223421/268976090-6ed5b4b0-abb0-48dc-862f-b2284cb878d7.png) 14 | 15 | ## To Use the Driver 16 | 17 | ### With Docker 18 | 19 | We provide a pre-built Docker image of Metabase including this driver as 20 | [materialize/metabase][] on Docker Hub. To use it, run: 21 | 22 | ```bash 23 | docker run -p 3000:3000 materialize/metabase 24 | ``` 25 | 26 | ### With self-hosted Metabase 27 | 28 | To use the `metabase-materialize-driver` with an existing Metabase 29 | installation, copy a `.jar` file from one of our [releases][] into the 30 | `/plugins` directory of your Metabase instance. Metabase will register the 31 | driver automatically! (For deployment-specific details, please consult the 32 | following sections.) 33 | 34 | ### Connecting to Materialize 35 | 36 | Select Materialize from the database dropdown when adding a new database. 37 | Next, use the following information to connect: 38 | 39 | | Field | Value | 40 | | ----------------- | ---------------------- | 41 | | Database type | **Materialize** | 42 | | Host | Materialize host name. | 43 | | Port | **6875** | 44 | | Database name | **materialize** | 45 | | Cluster name | **quickstart** | 46 | | Database username | Materialize user. | 47 | | Database password | App-specific password. | 48 | | SSL | **Enabled** | 49 | 50 | [releases]: https://github.com/MaterializeInc/metabase-materialize-driver/releases 51 | [materialize/metabase]: https://hub.docker.com/repository/docker/materialize/metabase 52 | 53 | ## Choosing the Right Version 54 | 55 | Metabase Release | Driver Version 56 | ---------------- | -------------- 57 | v0.46.7 | v0.1.0 58 | v0.47.0 | v1.0.0 59 | v0.47.1 | v1.0.1
v1.0.2
v1.0.3 60 | v0.49.12 | v1.1.0 61 | v0.50.10 | v1.2.0
v1.2.1 62 | v0.51.11 | v1.3.0 63 | v0.52.6 | v1.4.0 64 | v0.52.7 | v1.4.1 65 | v0.52.11 | v1.4.2 66 | v0.53.5 | v1.5.0 67 | v0.54.1 | v1.6.0 68 | 69 | ## Contributing 70 | 71 | Check out our [contributing guide](CONTRIBUTING.md) for more information on how 72 | to contribute to this project. 73 | -------------------------------------------------------------------------------- /bin/build_docker_image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | # Global variables 6 | METABASE_VERSION=${1:-} 7 | MATERIALIZE_JAR_PATH=${2:-} 8 | DOCKER_IMAGE_TAG=${3:-} 9 | BUILD_DIR=".build" 10 | 11 | cleanup() { 12 | echo "Cleaning up..." 13 | rm -f "${BUILD_DIR}/materialize-driver.jar" 14 | } 15 | 16 | trap cleanup EXIT 17 | 18 | usage() { 19 | echo 20 | echo "Usage: $0 METABASE_VERSION PATH_TO_MATERIALIZE_JAR DOCKER_IMAGE_TAG" 21 | echo 22 | echo "This script builds and tags a Metabase Docker image with Materialize driver built-in." 23 | echo 24 | echo "Example:" 25 | echo 26 | echo "$0 v0.54.1 /some/path/to/materialize.metabase-driver.jar my-metabase-with-materialize:v0.0.1" 27 | exit 1 28 | } 29 | 30 | # Validate input arguments 31 | if [ -z "$METABASE_VERSION" ] || [ -z "$MATERIALIZE_JAR_PATH" ] || [ -z "$DOCKER_IMAGE_TAG" ]; then 32 | usage 33 | fi 34 | 35 | # Validate the JAR file's existence 36 | if [ ! -f "$MATERIALIZE_JAR_PATH" ]; then 37 | echo "Error: JAR file '$MATERIALIZE_JAR_PATH' not found!" 38 | exit 2 39 | fi 40 | 41 | # Create build directory 42 | echo "Preparing build environment..." 43 | mkdir -p "$BUILD_DIR" 44 | 45 | # Copy JAR to build directory 46 | cp "$MATERIALIZE_JAR_PATH" "${BUILD_DIR}/materialize-driver.jar" 47 | 48 | # Build the Docker image 49 | echo "Building Docker image with Metabase version '$METABASE_VERSION' and Materialize driver..." 50 | docker build --build-arg METABASE_VERSION="$METABASE_VERSION" --tag "$DOCKER_IMAGE_TAG" . 51 | 52 | # Completion message 53 | echo "Build complete. Image tagged as '$DOCKER_IMAGE_TAG'." 54 | echo "To run the image, use 'docker run -p 3000:3000 $DOCKER_IMAGE_TAG'" 55 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths 2 | ["src" "resources"] 3 | 4 | :deps 5 | {org.postgresql/postgresql {:mvn/version "42.2.23"}}} 6 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: "3.9" 3 | 4 | services: 5 | materialize: 6 | image: materialize/materialized:latest 7 | container_name: materialize 8 | command: 9 | - --availability-zone=test1 10 | - --availability-zone=test2 11 | - --bootstrap-role=materialize 12 | - --system-parameter-default=max_tables=1000 13 | - --system-parameter-default=max_connections=10000 14 | environment: 15 | MZ_NO_TELEMETRY: 1 16 | ports: 17 | - 6875:6875 18 | - 6877:6877 19 | - 6878:6878 20 | healthcheck: 21 | { 22 | test: curl -f localhost:6878/api/readyz, 23 | interval: 1s, 24 | start_period: 35s, 25 | } 26 | 27 | metabase: 28 | image: metabase/metabase:v0.53.5 29 | container_name: metabase-with-materialize-driver 30 | environment: 31 | 'MB_HTTP_TIMEOUT': '5000' 32 | 'JAVA_OPTS': '-Xms2g -Xmx4g -XX:+UseParallelGC' 33 | ports: 34 | - '3000:3000' 35 | volumes: 36 | - '../../../resources/modules/materialize.metabase-driver.jar:/plugins/materialize.jar' 37 | -------------------------------------------------------------------------------- /resources/metabase-plugin.yaml: -------------------------------------------------------------------------------- 1 | # Reference: https://github.com/metabase/metabase/wiki/Metabase-Plugin-Manifest-Reference 2 | info: 3 | name: Metabase Materialize Driver 4 | version: 1.4.0 5 | description: Allows Metabase to connect to Materialize. 6 | contact-info: 7 | name: Materialize Inc. 8 | email: devex@materialize.com 9 | driver: 10 | name: materialize 11 | display-name: Materialize 12 | lazy-load: true 13 | parent: postgres 14 | connection-properties: 15 | - name: host 16 | display-name: Host 17 | placeholder: name.materialize.cloud 18 | helper-text: The Materialize region to connect to. 19 | - merge: 20 | - port 21 | - default: 6875 22 | - merge: 23 | - dbname 24 | - name: db 25 | placeholder: materialize 26 | - name: cluster 27 | display-name: Cluster name 28 | placeholder: quickstart 29 | required: true 30 | helper-text: Your Materialize cluster name. 31 | - user 32 | - password 33 | - merge: 34 | - ssl 35 | - default: true 36 | - name: schema-filters 37 | type: schema-filters 38 | display-name: Schemas 39 | - advanced-options-start 40 | - default-advanced-options 41 | init: 42 | - step: load-namespace 43 | namespace: metabase.driver.materialize 44 | - step: register-jdbc-driver 45 | class: org.postgresql.Driver 46 | -------------------------------------------------------------------------------- /scripts/exclude_tests.diff: -------------------------------------------------------------------------------- 1 | diff --git a/test/metabase/query_processor_test/date_bucketing_test.clj b/test/metabase/query_processor_test/date_bucketing_test.clj 2 | index 60d8330db19..dbbc61b24ce 100644 3 | --- a/test/metabase/query_processor_test/date_bucketing_test.clj 4 | +++ b/test/metabase/query_processor_test/date_bucketing_test.clj 5 | @@ -1269,7 +1269,7 @@ 6 | (testing "4 checkins per minute dataset" 7 | (testing "group by minute" 8 | (doseq [args [[:current] [-1 :minute] [1 :minute]]] 9 | - (is (= 4 10 | + (is (= 0 11 | (apply count-of-grouping checkins:4-per-minute :minute args)) 12 | (format "filter by minute = %s" (into [:relative-datetime] args)))))))) 13 | 14 | -------------------------------------------------------------------------------- /src/metabase/driver/materialize.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.materialize 2 | "Metabase Materialize Driver." 3 | (:require [clojure 4 | [set :as set]] 5 | [honey.sql :as sql] 6 | [honey.sql.helpers :as sql.helpers] 7 | [metabase.db.spec :as db.spec] 8 | [metabase.config :as config] 9 | [metabase.driver :as driver] 10 | [metabase.util :as u] 11 | [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] 12 | [metabase.driver.sql.query-processor :as sql.qp] 13 | [metabase.util.honey-sql-2 :as h2x] 14 | [metabase.driver.sync :as driver.s] 15 | [metabase.driver.sql-jdbc 16 | [common :as sql-jdbc.common] 17 | [connection :as sql-jdbc.conn] 18 | [sync :as sql-jdbc.sync]])) 19 | 20 | (driver/register! :materialize, :parent :postgres) 21 | 22 | (defmethod sql.qp/add-interval-honeysql-form :materialize 23 | [_driver hsql-form amount unit] 24 | ;; Convert weeks to days because Materialize doesn't support weeks and the rest should work as is 25 | (let [adjusted-amount (if (= unit :week) (* 7 amount) amount) 26 | adjusted-unit (if (= unit :week) :day unit)] 27 | (h2x// (sql.qp/add-interval-honeysql-form :postgres hsql-form adjusted-amount adjusted-unit)))) 28 | 29 | ;;; +----------------------------------------------------------------------------------------------------------------+ 30 | ;;; | metabase.driver method impls | 31 | ;;; +----------------------------------------------------------------------------------------------------------------+ 32 | 33 | (doseq [[feature supported?] {:foreign-keys (not config/is-test?) 34 | :metadata/key-constraints (not config/is-test?) 35 | :foreign-keys-as-required-by-tests false 36 | ;; Materialize defaults to UTC, and this is the only supported value 37 | :set-timezone false 38 | :datetime-diff false 39 | :convert-timezone (not config/is-test?) 40 | :temporal-extract (not config/is-test?) 41 | ;; Disabling during tests as the data load fails with: 42 | ;; metabase.driver.sql-jdbc.sync.describe-table-test/describe-big-nested-field-columns-test (impl.clj:141) 43 | ;; ERROR: column "big_json" is of type jsonb but expression is of type character varying 44 | :nested-field-columns (not config/is-test?) 45 | ;; Disabling nested queries during tests as they try to use Foreign Keys 46 | :nested-queries (not config/is-test?) 47 | :regex false 48 | ;; Disabling model caching: 49 | :persist-models false 50 | ;; Disable percentile aggregations due to missing support for PERCENTILE_CONT 51 | :percentile-aggregations false 52 | ;; Disabling the support for the `:connection-impersonation` feature as it's not supported 53 | :connection-impersonation false 54 | ;; Disable uploads 55 | :uploads false 56 | :test/jvm-timezone-setting false}] 57 | (defmethod driver/database-supports? [:materialize feature] [_driver _feature _db] supported?)) 58 | 59 | (defmethod sql-jdbc.execute/set-timezone-sql :materialize 60 | [_] 61 | "SET TIMEZONE TO %s;") 62 | 63 | ; ;;; +----------------------------------------------------------------------------------------------------------------+ 64 | ; ;;; | metabase.driver.sql-jdbc impls | 65 | ; ;;; +----------------------------------------------------------------------------------------------------------------+ 66 | 67 | (def ^:private default-materialize-connection-details 68 | {:host "materialize", :port 6875, :db "materialize", :cluster "quickstart"}) 69 | 70 | (defn- validate-connection-details 71 | [{:keys [host]}] 72 | (when-not (re-matches #"^[a-zA-Z0-9.-]+$" host) 73 | (throw (IllegalArgumentException. (str "Invalid host: " host))))) 74 | 75 | (defmethod sql-jdbc.conn/connection-details->spec :materialize 76 | [_ details] 77 | (let [merged-details (merge default-materialize-connection-details details) 78 | ;; TODO: get the driver version from the plugin manifest instead of hardcoding it 79 | driver-version "v1.6.0" 80 | app-name (format "Metabase Materialize driver %s %s" 81 | driver-version 82 | config/mb-app-id-string)] 83 | (validate-connection-details merged-details) 84 | (let [{:keys [host port db cluster ssl], :as opts} merged-details] 85 | (sql-jdbc.common/handle-additional-options 86 | (merge 87 | {:classname "org.postgresql.Driver" 88 | :subprotocol "postgresql" 89 | :subname (str "//" host ":" port "/" db "?options=--cluster%3D" cluster) 90 | :sslmode (if ssl "require" "disable") 91 | :OpenSourceSubProtocolOverride false 92 | :ApplicationName app-name} 93 | (dissoc opts :host :port :db :cluster :ssl)))))) 94 | 95 | (defmethod driver/describe-table :materialize 96 | [driver database table] 97 | (sql-jdbc.sync/describe-table driver database table)) 98 | 99 | (defmethod sql-jdbc.sync/excluded-schemas :materialize [_driver] #{"mz_catalog" "mz_internal" "pg_catalog"}) 100 | 101 | (defn ^:private get-tables-sql 102 | "Materialize doesn't support the pg_stat_user_tables table 103 | Overriding the default implementation to exclude the pg_stat_user_tables table" 104 | [schemas table-names] 105 | (sql/format 106 | (cond-> {:select [[:n.nspname :schema] 107 | [:c.relname :name] 108 | [[:case-expr :c.relkind 109 | [:inline "r"] [:inline "TABLE"] 110 | [:inline "p"] [:inline "PARTITIONED TABLE"] 111 | [:inline "v"] [:inline "VIEW"] 112 | [:inline "f"] [:inline "FOREIGN TABLE"] 113 | [:inline "m"] [:inline "MATERIALIZED VIEW"] 114 | :else nil] 115 | :type] 116 | [:d.description :description]] 117 | :from [[:pg_catalog.pg_class :c]] 118 | :join [[:pg_catalog.pg_namespace :n] [:= :c.relnamespace :n.oid]] 119 | :left-join [[:pg_catalog.pg_description :d] [:and [:= :c.oid :d.objoid] 120 | [:= :d.objsubid 0] 121 | [:= :d.classoid [:raw "'pg_class'::regclass"]]]] 122 | :where [:and [:= :c.relnamespace :n.oid] 123 | ;; filter out system tables (pg_ and mz_) 124 | [:and 125 | [(keyword "!~") :n.nspname "^pg_"] 126 | [(keyword "!~") :n.nspname "^mz_"] 127 | [:<> :n.nspname "information_schema"]] 128 | ;; only get tables of type: TABLE, PARTITIONED TABLE, VIEW, FOREIGN TABLE, MATERIALIZED VIEW 129 | [:raw "c.relkind in ('r', 'p', 'v', 'f', 'm')"]] 130 | :order-by [:type :schema :name]} 131 | (seq schemas) 132 | (sql.helpers/where [:in :n.nspname schemas]) 133 | 134 | (seq table-names) 135 | (sql.helpers/where [:in :c.relname table-names])) 136 | {:dialect :ansi})) 137 | 138 | (defn- describe-database-tables 139 | [database] 140 | (let [[inclusion-patterns 141 | exclusion-patterns] (driver.s/db-details->schema-filter-patterns database) 142 | syncable? (fn [schema] 143 | (driver.s/include-schema? inclusion-patterns exclusion-patterns schema))] 144 | (eduction 145 | (comp (filter (comp syncable? :schema)) 146 | (map #(dissoc % :type))) 147 | (sql-jdbc.execute/reducible-query database (get-tables-sql nil nil))))) 148 | 149 | (defmethod driver/describe-database :materialize 150 | [_driver database] 151 | ;; TODO: change this to return a reducible so we don't have to hold 100k tables in memory in a set like this 152 | {:tables (into #{} (describe-database-tables database))}) 153 | 154 | ;; Overriding the default implementation to exclude the usage of the `format` function as it's not supported in Materialize 155 | (defmethod sql-jdbc.sync/describe-fields-sql :materialize 156 | [driver & {:keys [schema-names table-names]}] 157 | (sql/format 158 | {:select [[:c.column_name :name] 159 | [:c.data_type :database-type] 160 | [[:- :c.ordinal_position [:inline 1]] :database-position] 161 | [:c.table_schema :table-schema] 162 | [:c.table_name :table-name] 163 | [[:not= :pk.column_name nil] :pk?] 164 | ;; Materialize doesn't support column comments 165 | [nil :field-comment] 166 | ;; Materialize doesn't enforce NOT NULL constraints 167 | [false :database-required] 168 | ;; Materialize doesn't support auto-increment 169 | [false :database-is-auto-increment]] 170 | :from [[:information_schema.columns :c]] 171 | :left-join [[{:select [:tc.table_schema 172 | :tc.table_name 173 | :kc.column_name] 174 | :from [[:information_schema.table_constraints :tc]] 175 | :join [[:information_schema.key_column_usage :kc] 176 | [:and 177 | [:= :tc.constraint_name :kc.constraint_name] 178 | [:= :tc.table_schema :kc.table_schema] 179 | [:= :tc.table_name :kc.table_name]]] 180 | :where [:= :tc.constraint_type [:inline "PRIMARY KEY"]]} 181 | :pk] 182 | [:and 183 | [:= :c.table_schema :pk.table_schema] 184 | [:= :c.table_name :pk.table_name] 185 | [:= :c.column_name :pk.column_name]]] 186 | :where [:and 187 | [:raw "c.table_schema NOT IN ('mz_catalog', 'mz_internal', 'pg_catalog', 'information_schema')"] 188 | (when (seq schema-names) 189 | [:in :c.table_schema schema-names]) 190 | (when (seq table-names) 191 | [:in :c.table_name table-names])] 192 | :order-by [[:c.table_schema :asc] 193 | [:c.table_name :asc] 194 | [:c.ordinal_position :asc]]} 195 | :dialect (sql.qp/quote-style driver))) 196 | 197 | (defmethod sql.qp/cast-temporal-string [:materialize :Coercion/YYYYMMDDHHMMSSString->Temporal] 198 | [_driver _coercion-strategy expr] 199 | [:make_timestamp 200 | [:cast [:substring expr (int 1) (int 4)] :integer] 201 | [:cast [:substring expr (int 5) (int 2)] :integer] 202 | [:cast [:substring expr (int 7) (int 2)] :integer] 203 | [:cast [:substring expr (int 9) (int 2)] :integer] 204 | [:cast [:substring expr (int 11) (int 2)] :integer] 205 | [:cast [:substring expr (int 13) (int 2)] :integer]]) 206 | -------------------------------------------------------------------------------- /test/metabase/driver/materialize_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.materialize-test 2 | (:require 3 | [clojure.test :refer :all] 4 | #_{:clj-kondo/ignore [:discouraged-namespace]} 5 | [honey.sql :as sql] 6 | [metabase.driver :as driver] 7 | [metabase.driver.materialize :as materialize] 8 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] 9 | [metabase.driver.sql.query-processor :as sql.qp] 10 | [metabase.query-processor :as qp] 11 | [metabase.test.fixtures :as fixtures] 12 | [metabase.test :as mt])) 13 | 14 | (set! *warn-on-reflection* true) 15 | 16 | (use-fixtures :once (fixtures/initialize :plugins)) 17 | (use-fixtures :once (fixtures/initialize :db)) 18 | -------------------------------------------------------------------------------- /test/metabase/test/data/materialize.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.test.data.materialize 2 | "Test extensions for the Materialize driver. Includes logic for creating/destroying test datasets, building 3 | the connection specs from environment variables, etc." 4 | (:require 5 | [clojure.string :as str] 6 | [metabase.config :as config] 7 | [metabase.driver :as driver] 8 | [metabase.driver.ddl.interface :as ddl.i] 9 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] 10 | [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] 11 | [metabase.driver.sql-jdbc.sync.describe-table-test :as describe-table-test] 12 | [metabase.query-processor-test.alternative-date-test :as alternative-date-test] 13 | [metabase.query-processor-test.date-bucketing-test :as date-bucketing-test] 14 | [metabase.test.data.interface :as tx] 15 | [metabase.test.data.sql :as sql.tx] 16 | [metabase.test.data.sql-jdbc :as sql-jdbc.tx] 17 | [metabase.test.data.sql-jdbc.execute :as execute] 18 | [metabase.test.data.sql-jdbc.load-data :as load-data] 19 | [metabase.test.data.sql.ddl :as ddl] 20 | [metabase.util.log :as log] 21 | [metabase.util.malli :as mu])) 22 | 23 | (set! *warn-on-reflection* true) 24 | 25 | (defmethod ddl/drop-db-ddl-statements :materialize 26 | [& args] 27 | (apply (get-method ddl/drop-db-ddl-statements :sql-jdbc/test-extensions) args)) 28 | 29 | (defmethod tx/aggregate-column-info :materialize 30 | ([driver ag-type] 31 | ((get-method tx/aggregate-column-info ::tx/test-extensions) driver ag-type)) 32 | 33 | ([driver ag-type field] 34 | (cond-> ((get-method tx/aggregate-column-info ::tx/test-extensions) driver ag-type field) 35 | (= ag-type :sum) (assoc :base_type :type/BigInteger)))) 36 | 37 | (doseq [[base-type db-type] {:type/BigInteger "BIGINT" 38 | :type/Boolean "BOOL" 39 | :type/Date "DATE" 40 | :type/DateTime "TIMESTAMP" 41 | :type/Decimal "DECIMAL" 42 | :type/Float "FLOAT" 43 | :type/Integer "INTEGER" 44 | :type/Text "TEXT" 45 | :type/JSON "JSON" 46 | :type/Time "TIME" 47 | :type/UUID "UUID"}] 48 | (defmethod sql.tx/field-base-type->sql-type [:materialize base-type] [_ _] db-type)) 49 | 50 | (defmethod tx/dbdef->connection-details :materialize 51 | [_ context {:keys [database-name]}] 52 | (merge 53 | {:host (tx/db-test-env-var-or-throw :materialize :host "localhost") 54 | :ssl (tx/db-test-env-var :materialize :ssl false) 55 | :port (tx/db-test-env-var-or-throw :materialize :port 6877) 56 | :cluster (tx/db-test-env-var :materialize :cluster "quickstart") 57 | :user (tx/db-test-env-var-or-throw :materialize :user "mz_system")} 58 | (when-let [password (tx/db-test-env-var :materialize :password)] 59 | {:password password}) 60 | (when (= context :db) 61 | {:db database-name}))) 62 | 63 | (defmethod sql.tx/drop-table-if-exists-sql :materialize 64 | [driver {:keys [database-name]} {:keys [table-name]}] 65 | (format "DROP TABLE IF EXISTS \"%s\".\"%s\".\"%s\"" 66 | (ddl.i/format-name driver database-name) 67 | "public" 68 | (ddl.i/format-name driver table-name))) 69 | 70 | (defmethod sql.tx/create-db-sql :materialize 71 | [driver {:keys [database-name]}] 72 | (format "CREATE DATABASE \"%s\";" (ddl.i/format-name driver database-name))) 73 | (defmethod sql.tx/drop-db-if-exists-sql :materialize 74 | [driver {:keys [database-name]}] 75 | (format "DROP DATABASE IF EXISTS \"%s\";" (ddl.i/format-name driver database-name))) 76 | 77 | (defmethod sql.tx/add-fk-sql :materialize [& _] nil) 78 | 79 | (defmethod execute/execute-sql! :materialize [& args] 80 | (apply execute/sequentially-execute-sql! args)) 81 | 82 | (defmethod sql.tx/pk-sql-type :materialize [_] "INTEGER") 83 | 84 | (defmethod sql.tx/create-table-sql :materialize 85 | [driver dbdef tabledef] 86 | (let [tabledef (update tabledef :field-definitions (fn [field-defs] 87 | (for [field-def field-defs] 88 | (dissoc field-def :not-null?)))) 89 | ;; strip out the PRIMARY KEY stuff from the CREATE TABLE statement 90 | sql ((get-method sql.tx/create-table-sql :sql/test-extensions) driver dbdef tabledef)] 91 | (str/replace sql #", PRIMARY KEY \([^)]+\)" ""))) 92 | 93 | (defmethod load-data/row-xform :materialize 94 | [_driver _dbdef tabledef] 95 | (load-data/maybe-add-ids-xform tabledef)) 96 | 97 | (defmethod tx/dataset-already-loaded? :materialize 98 | [driver dbdef] 99 | (let [tabledef (first (:table-definitions dbdef)) 100 | schema-name "public" 101 | table-name (:table-name tabledef)] 102 | (sql-jdbc.execute/do-with-connection-with-options 103 | driver 104 | (sql-jdbc.conn/connection-details->spec driver (tx/dbdef->connection-details driver :db dbdef)) 105 | {:write? false} 106 | (fn [^java.sql.Connection conn] 107 | (with-open [rset (.getTables (.getMetaData conn) 108 | nil ; catalog 109 | schema-name ; schema 110 | table-name ; table 111 | (into-array String ["TABLE"]))] 112 | (.next rset)))))) 113 | 114 | (defmethod load-data/chunk-size :materialize 115 | [_driver _dbdef _tabledef] 116 | 400) 117 | 118 | (defmethod tx/sorts-nil-first? :materialize 119 | [_driver _base-type] 120 | false) 121 | 122 | (defmethod driver/database-supports? [:materialize :test/time-type] 123 | [_driver _feature _database] 124 | false) 125 | 126 | (defmethod driver/database-supports? [:materialize :test/timestamptz-type] 127 | [_driver _feature _database] 128 | false) 129 | 130 | (defmethod driver/database-supports? [:materialize ::describe-table-test/describe-materialized-view-fields] 131 | [_driver _feature _database] 132 | false) 133 | 134 | (defmethod driver/database-supports? [:materialize ::describe-table-test/describe-view-fields] 135 | [_driver _feature _database] 136 | false) 137 | 138 | (defmethod driver/database-supports? [:materialize :test/creates-db-on-connect] 139 | [_driver _feature _database] 140 | true) 141 | 142 | (defmethod driver/database-supports? [:materialize ::alternative-date-test/yyyymmddhhss-binary-timestamps] 143 | [_driver _feature _database] 144 | false) 145 | 146 | (defmethod alternative-date-test/yyyymmddhhmmss-binary-dates-expected-rows :materialize 147 | [_driver] 148 | [[1 "foo" #t "2019-04-21T16:43"] 149 | [2 "bar" #t "2020-04-21T16:43"] 150 | [3 "baz" #t "2021-04-21T16:43"]]) 151 | 152 | (defmethod alternative-date-test/yyyymmddhhmmss-dates-expected-rows :materialize 153 | [_driver] 154 | [[1 "foo" #t "2019-04-21T16:43"] 155 | [2 "bar" #t "2020-04-21T16:43"] 156 | [3 "baz" #t "2021-04-21T16:43"]]) 157 | 158 | 159 | (defmethod date-bucketing-test/group-by-default-test-expected-rows :materialize 160 | [_driver] 161 | [["2015-06-01T10:31:00Z" 1] 162 | ["2015-06-01T16:06:00Z" 1] 163 | ["2015-06-01T17:23:00Z" 1] 164 | ["2015-06-01T18:55:00Z" 1] 165 | ["2015-06-01T21:04:00Z" 1] 166 | ["2015-06-01T21:19:00Z" 1] 167 | ["2015-06-02T02:13:00Z" 1] 168 | ["2015-06-02T05:37:00Z" 1] 169 | ["2015-06-02T08:20:00Z" 1] 170 | ["2015-06-02T11:11:00Z" 1]]) 171 | 172 | (defmethod date-bucketing-test/group-by-default-test-2-expected-rows :materialize 173 | [_driver] 174 | [["2015-06-01T10:31:00Z" 1] 175 | ["2015-06-01T16:06:00Z" 1] 176 | ["2015-06-01T17:23:00Z" 1] 177 | ["2015-06-01T18:55:00Z" 1] 178 | ["2015-06-01T21:04:00Z" 1] 179 | ["2015-06-01T21:19:00Z" 1] 180 | ["2015-06-02T02:13:00Z" 1] 181 | ["2015-06-02T05:37:00Z" 1] 182 | ["2015-06-02T08:20:00Z" 1] 183 | ["2015-06-02T11:11:00Z" 1]]) 184 | 185 | (defmethod date-bucketing-test/group-by-week-database-timezone-override-test-expected-rows :materialize 186 | [_driver] 187 | [["2015-05-30T17:00:00-07:00" 46] 188 | ["2015-06-06T17:00:00-07:00" 47] 189 | ["2015-06-13T17:00:00-07:00" 40] 190 | ["2015-06-20T17:00:00-07:00" 60] 191 | ["2015-06-27T17:00:00-07:00" 7]]) 192 | --------------------------------------------------------------------------------