├── .github
├── deps.edn
└── workflows
│ ├── dockerhub.yaml
│ ├── release.yaml
│ └── tests.yaml
├── .gitignore
├── CONTRIBUTING.md
├── Dockerfile
├── LICENSE
├── README.md
├── bin
└── build_docker_image.sh
├── deps.edn
├── docker-compose.yml
├── resources
└── metabase-plugin.yaml
├── scripts
└── exclude_tests.diff
├── src
└── metabase
│ └── driver
│ └── materialize.clj
└── test
└── metabase
├── driver
└── materialize_test.clj
└── test
└── data
└── materialize.clj
/.github/deps.edn:
--------------------------------------------------------------------------------
1 | {:aliases
2 | {:user/materialize
3 | {:extra-paths ["PWD/modules/drivers/materialize/test"]
4 | :extra-deps {metabase/materialize {:local/root "PWD/modules/drivers/materialize"}}}}}
5 |
--------------------------------------------------------------------------------
/.github/workflows/dockerhub.yaml:
--------------------------------------------------------------------------------
1 | name: Build and Publish Docker Image
2 |
3 | on:
4 | push:
5 | tags:
6 | - "v*"
7 | workflow_dispatch:
8 | inputs:
9 | version:
10 | description: "Version to build and publish"
11 | required: true
12 |
13 | jobs:
14 | dockerhub:
15 | runs-on: ubuntu-latest
16 | permissions:
17 | contents: write
18 | steps:
19 | - name: Checkout Metabase Repo
20 | uses: actions/checkout@v4
21 | with:
22 | repository: metabase/metabase
23 | ref: release-x.54.x
24 |
25 | - name: Checkout Driver Repo
26 | uses: actions/checkout@v4
27 | with:
28 | path: modules/drivers/materialize
29 |
30 | - name: Set up JDK 21
31 | uses: actions/setup-java@v2
32 | with:
33 | distribution: temurin
34 | java-version: 21
35 |
36 | - name: Install Clojure CLI
37 | run: |
38 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1262.sh &&
39 | sudo bash ./linux-install-1.11.1.1262.sh
40 |
41 | - name: Setup Node
42 | uses: actions/setup-node@v2
43 | with:
44 | node-version: "22"
45 | cache: "yarn"
46 |
47 | - name: Get M2 cache
48 | uses: actions/cache@v4
49 | with:
50 | path: |
51 | ~/.m2
52 | ~/.gitlibs
53 | key: ${{ runner.os }}-materialize-${{ hashFiles('**/deps.edn') }}
54 |
55 | - name: Prepare stuff for pulses
56 | run: yarn build-static-viz
57 |
58 | - name: Build Materialize driver
59 | run: |
60 | echo "{:deps {metabase/materialize {:local/root \"materialize\" }}}" > modules/drivers/deps.edn
61 | bin/build-driver.sh materialize
62 | mkdir -p modules/drivers/materialize/.build
63 | cp resources/modules/materialize.metabase-driver.jar modules/drivers/materialize/.build/materialize-driver.jar
64 |
65 | - name: Use QEMU for multi-platform images
66 | uses: docker/setup-qemu-action@v2
67 |
68 | - name: Use buildx for multi-platform images
69 | uses: docker/setup-buildx-action@v2
70 |
71 | - name: Log in to Docker Hub
72 | uses: docker/login-action@v2
73 | with:
74 | username: materializebot
75 | password: ${{ secrets.DOCKER_HUB_MATERIALIZEBOT_API_KEY }}
76 |
77 | - name: Extract metadata (tags, labels) for Docker
78 | id: meta
79 | uses: docker/metadata-action@v4
80 | with:
81 | images: materialize/metabase
82 | tags: |
83 | type=semver,pattern={{version}}
84 |
85 | - name: Build and push Docker image
86 | uses: docker/build-push-action@v4
87 | with:
88 | context: modules/drivers/materialize
89 | platforms: linux/amd64,linux/arm64
90 | push: true
91 | tags: ${{ steps.meta.outputs.tags }}
92 | labels: ${{ steps.meta.outputs.labels }}
93 |
--------------------------------------------------------------------------------
/.github/workflows/release.yaml:
--------------------------------------------------------------------------------
1 | name: Release
2 | on:
3 | push:
4 | tags:
5 | - "v*"
6 |
7 | jobs:
8 | release:
9 | runs-on: ubuntu-latest
10 | permissions:
11 | contents: write
12 | steps:
13 | - name: Checkout Metabase Repo
14 | uses: actions/checkout@v4
15 | with:
16 | repository: metabase/metabase
17 | ref: release-x.54.x
18 |
19 | - name: Checkout Driver Repo
20 | uses: actions/checkout@v4
21 | with:
22 | path: modules/drivers/materialize
23 |
24 | - name: Set up JDK 21
25 | uses: actions/setup-java@v2
26 | with:
27 | distribution: temurin
28 | java-version: 21
29 |
30 | - name: Install Clojure CLI
31 | run: |
32 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1262.sh &&
33 | sudo bash ./linux-install-1.11.1.1262.sh
34 |
35 | - name: Setup Node
36 | uses: actions/setup-node@v2
37 | with:
38 | node-version: "22"
39 | cache: "yarn"
40 |
41 | - name: Get M2 cache
42 | uses: actions/cache@v4
43 | with:
44 | path: |
45 | ~/.m2
46 | ~/.gitlibs
47 | key: ${{ runner.os }}-materialize-${{ hashFiles('**/deps.edn') }}
48 |
49 | - name: Prepare stuff for pulses
50 | run: yarn build-static-viz
51 |
52 | - name: Build Materialize driver
53 | run: |
54 | echo "{:deps {metabase/materialize {:local/root \"materialize\" }}}" > modules/drivers/deps.edn
55 | bin/build-driver.sh materialize
56 | ls -lah resources/modules
57 |
58 | - name: Install GitHub CLI
59 | run: |
60 | sudo apt update -y
61 | sudo apt install gh -y
62 |
63 | - name: Setup GitHub CLI
64 | working-directory: modules/drivers/materialize
65 | run: |
66 | gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}"
67 |
68 | - name: Create Release
69 | working-directory: modules/drivers/materialize
70 | run: |
71 | gh release create ${{ github.ref }} ../../../resources/modules/materialize.metabase-driver.jar -t "Release ${{ github.ref }}" -n "Release of the Materialize driver for Metabase"
72 |
--------------------------------------------------------------------------------
/.github/workflows/tests.yaml:
--------------------------------------------------------------------------------
1 | name: Tests
2 |
3 | on:
4 | workflow_dispatch:
5 | pull_request:
6 | branches:
7 | - master
8 | paths-ignore:
9 | - "**.md"
10 | schedule:
11 | - cron: 0 11 * * 0
12 |
13 | jobs:
14 | test:
15 | runs-on: ubuntu-latest
16 | steps:
17 | - name: Checkout Metabase Repo
18 | uses: actions/checkout@v4
19 | with:
20 | repository: metabase/metabase
21 | ref: release-x.54.x
22 |
23 | - name: Checkout Driver Repo
24 | uses: actions/checkout@v4
25 | with:
26 | path: modules/drivers/materialize
27 |
28 | - name: Set up JDK 21
29 | uses: actions/setup-java@v2
30 | with:
31 | distribution: temurin
32 | java-version: 21
33 |
34 | - name: Add Materialize TLS instance to /etc/hosts
35 | run: |
36 | sudo echo "127.0.0.1 materialize" | sudo tee -a /etc/hosts
37 |
38 | - name: Start Materialize in Docker
39 | uses: hoverkraft-tech/compose-action@v2.0.1
40 | with:
41 | compose-file: "modules/drivers/materialize/docker-compose.yml"
42 | services: |
43 | materialize
44 |
45 | # Apply the scripts/exclude_tests.diff patch to exclude tests that are not relevant to Materialize
46 | - name: Apply exclude_tests.diff
47 | run: |
48 | git apply modules/drivers/materialize/scripts/exclude_tests.diff
49 |
50 | - name: Install Clojure CLI
51 | run: |
52 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1262.sh &&
53 | sudo bash ./linux-install-1.11.1.1262.sh
54 |
55 | - name: Setup Node
56 | uses: actions/setup-node@v2
57 | with:
58 | node-version: "22"
59 | cache: "yarn"
60 |
61 | - name: Get M2 cache
62 | uses: actions/cache@v4
63 | with:
64 | path: |
65 | ~/.m2
66 | ~/.gitlibs
67 | key: ${{ runner.os }}-materialize-${{ hashFiles('**/deps.edn') }}
68 |
69 | - name: Prepare stuff for pulses
70 | run: yarn build-static-viz
71 |
72 | # Use custom deps.edn containing "user/materialize" alias to include driver sources
73 | - name: Run tests
74 | run: |
75 | mkdir -p /home/runner/.config/clojure
76 | cat modules/drivers/materialize/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn
77 |
78 | # Retry tests up to 2 times as the Metabase test data sometimes fails to load on the first try
79 | ATTEMPTS=0
80 | MAX_RETRIES=2
81 |
82 | until [ $ATTEMPTS -ge $MAX_RETRIES ]
83 | do
84 | echo "Attempt $(($ATTEMPTS + 1)) of $MAX_RETRIES..."
85 | DRIVERS=materialize clojure -X:dev:drivers:drivers-dev:test:user/materialize && break
86 | ATTEMPTS=$(($ATTEMPTS + 1))
87 | echo "Tests failed. Retrying in 10 seconds..."
88 | sleep 10
89 | done
90 |
91 | if [ $ATTEMPTS -eq $MAX_RETRIES ]; then
92 | echo "Tests failed after $MAX_RETRIES attempts."
93 | exit 1
94 | fi
95 |
96 | - name: Build Materialize driver
97 | run: |
98 | echo "{:deps {metabase/materialize {:local/root \"materialize\" }}}" > modules/drivers/deps.edn
99 | bin/build-driver.sh materialize
100 | ls -lah resources/modules
101 |
102 | - name: Archive driver JAR
103 | uses: actions/upload-artifact@v4
104 | with:
105 | name: materialize.metabase-driver.jar
106 | path: resources/modules/materialize.metabase-driver.jar
107 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | .clj-kondo/
3 | .lsp/
4 | .cpcache/
5 | .build
6 | .DS_Store
7 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | ## Getting started
2 |
3 | * Please report any issues you encounter during operations.
4 | * Feel free to create a pull request, preferably with a test or five.
5 |
6 | ## Setting up a development environment
7 |
8 | ### Requirements
9 |
10 | * Clojure 1.11+
11 | * OpenJDK 21
12 | * Node.js 22.x
13 | * Yarn
14 |
15 | For testing: [Docker Compose](https://docs.docker.com/compose/install/)
16 |
17 | Please refer to the extensive documentation available on the Metabase website: [Guide to writing a Metabase driver](https://www.metabase.com/docs/latest/developers-guide/drivers/start.html)
18 |
19 | Materialize driver's code should be inside the main Metabase repository checkout in `modules/drivers/materialize` directory.
20 |
21 | The easiest way to set up a development environment is as follows (mostly the same as in the [CI](https://github.com/MaterializeInc/metabase-materialize-driver/blob/master/.github/workflows/tests.yml)):
22 |
23 | * Clone Metabase and Materialize driver repositories
24 | ```bash
25 | git clone https://github.com/metabase/metabase.git
26 | cd metabase
27 | git checkout release-x.54.x
28 | git clone https://github.com/MaterializeInc/metabase-materialize-driver.git modules/drivers/materialize
29 | ```
30 |
31 | * Create custom Clojure profiles, you can get it using the following command:
32 |
33 | ```bash
34 | cat modules/drivers/materialize/.github/deps.edn | sed -e "s|PWD|$PWD|g" | tr -d '\n'
35 | ```
36 |
37 | Modifying `~/.clojure/deps.edn` will create two useful profiles: `user/materialize` that adds driver's sources to the classpath, and `user/test` that includes all the Metabase tests that are guaranteed to work with the driver.
38 |
39 | * Install the Metabase dependencies:
40 |
41 | ```bash
42 | clojure -X:deps:drivers prep
43 | ```
44 |
45 | * Build the frontend:
46 |
47 | ```bash
48 | yarn && yarn build-static-viz
49 | ```
50 |
51 | * Add `/etc/hosts` entry
52 |
53 | Required for TLS tests.
54 |
55 | ```bash
56 | sudo -- sh -c "echo 127.0.0.1 materialize >> /etc/hosts"
57 | ```
58 |
59 | * Start Materialize as a Docker container
60 |
61 | ```bash
62 | docker compose -f modules/drivers/materialize/docker-compose.yml up -d materialize
63 | ```
64 |
65 | * To use a specific Java version, you can set the `JAVA_HOME` environment variable:
66 |
67 | ```bash
68 | export JAVA_HOME=$(/usr/libexec/java_home -v 21) ; export PATH=$JAVA_HOME/bin:$PATH
69 | ```
70 |
71 | Now, you should be able to run the tests:
72 |
73 | ```bash
74 | mz_deps=$(cat modules/drivers/materialize/.github/deps.edn | sed -e "s|PWD|$PWD|g" | tr -d '\n')
75 | DRIVERS=materialize clojure -Sdeps ${mz_deps} -X:dev:drivers:drivers-dev:test:user/materialize
76 | ```
77 |
78 | you can see that we have our profiles `:user/materialize:user/test` added to the command above, and with `DRIVERS=materialize` we instruct Metabase to run the tests only for Materialize.
79 |
80 | > **Note** Omitting `DRIVERS` will run the tests for all the built-in database drivers.
81 |
82 | If you want to run tests for only a specific test:
83 |
84 | ```bash
85 | mz_deps=$(cat modules/drivers/materialize/.github/deps.edn | sed -e "s|PWD|$PWD|g" | tr -d '\n')
86 | DRIVERS=materialize clojure -Sdeps ${mz_deps} -X:dev:drivers:drivers-dev:test:user/materialize :only metabase.query-processor.middleware.parameters.mbql-test
87 | ```
88 |
89 | ## Excluding tests
90 |
91 | Some tests are not applicable to Materialize, and you can exclude them by adding the following to the test command:
92 |
93 | ```bash
94 | git apply modules/drivers/materialize/scripts/exclude_tests.diff
95 | ```
96 |
97 | The diff file contains the list of tests that are excluded from the Materialize driver. This often needs to be updated as new tests are added to new Metabase versions.
98 |
99 | ## Building a jar
100 |
101 | You need to add an entry for Materialize in `modules/drivers/deps.edn`
102 |
103 | ```clj
104 | {:deps
105 | {...
106 | metabase/materialize {:local/root "materialize"}
107 | ...}}
108 | ```
109 |
110 | or just run this from the root Metabase directory, overwriting the entire file:
111 |
112 | ```bash
113 | echo "{:deps {metabase/materialize {:local/root \"materialize\" }}}" > modules/drivers/deps.edn
114 | ```
115 |
116 | Now, you should be able to build the final jar:
117 |
118 | ```bash
119 | bin/build-driver.sh materialize
120 | ```
121 |
122 | As the result, `resources/modules/materialize.metabase-driver.jar` should be created. You can copy it to the Metabase `plugins` directory and start Metabase.
123 |
124 | For smoke testing, there is a Metabase with the link to the driver available as a Docker container:
125 |
126 | ```bash
127 | docker compose -f modules/drivers/materialize/docker-compose.yml up -d metabase
128 | ```
129 |
130 | It should pick up the driver jar as a volume.
131 |
132 | ## Cutting a release
133 |
134 | Once the driver is ready to be released and tests are passing, you need to create a tag in the Metabase repository:
135 |
136 | ```bash
137 | git tag -a vX.Y.Z -m vX.Y.Z
138 | git push origin vX.Y.Z
139 | ```
140 |
141 | Once the tag is pushed, the CI will build the driver and create a release including the jar file. You can manually edit the release description to include any additional release notes.
142 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | ARG METABASE_VERSION=latest
2 | FROM metabase/metabase:${METABASE_VERSION}
3 |
4 | ADD .build/materialize-driver.jar /plugins/
5 | RUN chmod 744 /plugins/materialize-driver.jar
6 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 1997, PostgreSQL Global Development Group
2 | All rights reserved.
3 |
4 | Redistribution and use in source and binary forms, with or without
5 | modification, are permitted provided that the following conditions are met:
6 |
7 | 1. Redistributions of source code must retain the above copyright notice,
8 | this list of conditions and the following disclaimer.
9 | 2. Redistributions in binary form must reproduce the above copyright notice,
10 | this list of conditions and the following disclaimer in the documentation
11 | and/or other materials provided with the distribution.
12 |
13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
14 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
17 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
23 | POSSIBILITY OF SUCH DAMAGE.
24 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Materialize driver for Metabase
2 |
3 | [](https://materialize.com/s/chat)
4 |
5 | The `metabase-materialize-driver` lets
6 | [Metabase](https://github.com/metabase/metabase) connect to an instance of
7 | [Materialize](https://github.com/MaterializeInc/materialize).
8 |
9 | > [!NOTE]
10 | > With improvements to Materialize's PostgreSQL compatibility, you can now connect to Materialize directly using Metabase's default PostgreSQL driver.
11 | > See the guide here: [Using Materialize with Metabase](https://materialize.com/docs/serve-results/metabase/).
12 |
13 | 
14 |
15 | ## To Use the Driver
16 |
17 | ### With Docker
18 |
19 | We provide a pre-built Docker image of Metabase including this driver as
20 | [materialize/metabase][] on Docker Hub. To use it, run:
21 |
22 | ```bash
23 | docker run -p 3000:3000 materialize/metabase
24 | ```
25 |
26 | ### With self-hosted Metabase
27 |
28 | To use the `metabase-materialize-driver` with an existing Metabase
29 | installation, copy a `.jar` file from one of our [releases][] into the
30 | `/plugins` directory of your Metabase instance. Metabase will register the
31 | driver automatically! (For deployment-specific details, please consult the
32 | following sections.)
33 |
34 | ### Connecting to Materialize
35 |
36 | Select Materialize from the database dropdown when adding a new database.
37 | Next, use the following information to connect:
38 |
39 | | Field | Value |
40 | | ----------------- | ---------------------- |
41 | | Database type | **Materialize** |
42 | | Host | Materialize host name. |
43 | | Port | **6875** |
44 | | Database name | **materialize** |
45 | | Cluster name | **quickstart** |
46 | | Database username | Materialize user. |
47 | | Database password | App-specific password. |
48 | | SSL | **Enabled** |
49 |
50 | [releases]: https://github.com/MaterializeInc/metabase-materialize-driver/releases
51 | [materialize/metabase]: https://hub.docker.com/repository/docker/materialize/metabase
52 |
53 | ## Choosing the Right Version
54 |
55 | Metabase Release | Driver Version
56 | ---------------- | --------------
57 | v0.46.7 | v0.1.0
58 | v0.47.0 | v1.0.0
59 | v0.47.1 | v1.0.1
v1.0.2
v1.0.3
60 | v0.49.12 | v1.1.0
61 | v0.50.10 | v1.2.0
v1.2.1
62 | v0.51.11 | v1.3.0
63 | v0.52.6 | v1.4.0
64 | v0.52.7 | v1.4.1
65 | v0.52.11 | v1.4.2
66 | v0.53.5 | v1.5.0
67 | v0.54.1 | v1.6.0
68 |
69 | ## Contributing
70 |
71 | Check out our [contributing guide](CONTRIBUTING.md) for more information on how
72 | to contribute to this project.
73 |
--------------------------------------------------------------------------------
/bin/build_docker_image.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | set -euo pipefail
4 |
5 | # Global variables
6 | METABASE_VERSION=${1:-}
7 | MATERIALIZE_JAR_PATH=${2:-}
8 | DOCKER_IMAGE_TAG=${3:-}
9 | BUILD_DIR=".build"
10 |
11 | cleanup() {
12 | echo "Cleaning up..."
13 | rm -f "${BUILD_DIR}/materialize-driver.jar"
14 | }
15 |
16 | trap cleanup EXIT
17 |
18 | usage() {
19 | echo
20 | echo "Usage: $0 METABASE_VERSION PATH_TO_MATERIALIZE_JAR DOCKER_IMAGE_TAG"
21 | echo
22 | echo "This script builds and tags a Metabase Docker image with Materialize driver built-in."
23 | echo
24 | echo "Example:"
25 | echo
26 | echo "$0 v0.54.1 /some/path/to/materialize.metabase-driver.jar my-metabase-with-materialize:v0.0.1"
27 | exit 1
28 | }
29 |
30 | # Validate input arguments
31 | if [ -z "$METABASE_VERSION" ] || [ -z "$MATERIALIZE_JAR_PATH" ] || [ -z "$DOCKER_IMAGE_TAG" ]; then
32 | usage
33 | fi
34 |
35 | # Validate the JAR file's existence
36 | if [ ! -f "$MATERIALIZE_JAR_PATH" ]; then
37 | echo "Error: JAR file '$MATERIALIZE_JAR_PATH' not found!"
38 | exit 2
39 | fi
40 |
41 | # Create build directory
42 | echo "Preparing build environment..."
43 | mkdir -p "$BUILD_DIR"
44 |
45 | # Copy JAR to build directory
46 | cp "$MATERIALIZE_JAR_PATH" "${BUILD_DIR}/materialize-driver.jar"
47 |
48 | # Build the Docker image
49 | echo "Building Docker image with Metabase version '$METABASE_VERSION' and Materialize driver..."
50 | docker build --build-arg METABASE_VERSION="$METABASE_VERSION" --tag "$DOCKER_IMAGE_TAG" .
51 |
52 | # Completion message
53 | echo "Build complete. Image tagged as '$DOCKER_IMAGE_TAG'."
54 | echo "To run the image, use 'docker run -p 3000:3000 $DOCKER_IMAGE_TAG'"
55 |
--------------------------------------------------------------------------------
/deps.edn:
--------------------------------------------------------------------------------
1 | {:paths
2 | ["src" "resources"]
3 |
4 | :deps
5 | {org.postgresql/postgresql {:mvn/version "42.2.23"}}}
6 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | ---
2 | version: "3.9"
3 |
4 | services:
5 | materialize:
6 | image: materialize/materialized:latest
7 | container_name: materialize
8 | command:
9 | - --availability-zone=test1
10 | - --availability-zone=test2
11 | - --bootstrap-role=materialize
12 | - --system-parameter-default=max_tables=1000
13 | - --system-parameter-default=max_connections=10000
14 | environment:
15 | MZ_NO_TELEMETRY: 1
16 | ports:
17 | - 6875:6875
18 | - 6877:6877
19 | - 6878:6878
20 | healthcheck:
21 | {
22 | test: curl -f localhost:6878/api/readyz,
23 | interval: 1s,
24 | start_period: 35s,
25 | }
26 |
27 | metabase:
28 | image: metabase/metabase:v0.53.5
29 | container_name: metabase-with-materialize-driver
30 | environment:
31 | 'MB_HTTP_TIMEOUT': '5000'
32 | 'JAVA_OPTS': '-Xms2g -Xmx4g -XX:+UseParallelGC'
33 | ports:
34 | - '3000:3000'
35 | volumes:
36 | - '../../../resources/modules/materialize.metabase-driver.jar:/plugins/materialize.jar'
37 |
--------------------------------------------------------------------------------
/resources/metabase-plugin.yaml:
--------------------------------------------------------------------------------
1 | # Reference: https://github.com/metabase/metabase/wiki/Metabase-Plugin-Manifest-Reference
2 | info:
3 | name: Metabase Materialize Driver
4 | version: 1.4.0
5 | description: Allows Metabase to connect to Materialize.
6 | contact-info:
7 | name: Materialize Inc.
8 | email: devex@materialize.com
9 | driver:
10 | name: materialize
11 | display-name: Materialize
12 | lazy-load: true
13 | parent: postgres
14 | connection-properties:
15 | - name: host
16 | display-name: Host
17 | placeholder: name.materialize.cloud
18 | helper-text: The Materialize region to connect to.
19 | - merge:
20 | - port
21 | - default: 6875
22 | - merge:
23 | - dbname
24 | - name: db
25 | placeholder: materialize
26 | - name: cluster
27 | display-name: Cluster name
28 | placeholder: quickstart
29 | required: true
30 | helper-text: Your Materialize cluster name.
31 | - user
32 | - password
33 | - merge:
34 | - ssl
35 | - default: true
36 | - name: schema-filters
37 | type: schema-filters
38 | display-name: Schemas
39 | - advanced-options-start
40 | - default-advanced-options
41 | init:
42 | - step: load-namespace
43 | namespace: metabase.driver.materialize
44 | - step: register-jdbc-driver
45 | class: org.postgresql.Driver
46 |
--------------------------------------------------------------------------------
/scripts/exclude_tests.diff:
--------------------------------------------------------------------------------
1 | diff --git a/test/metabase/query_processor_test/date_bucketing_test.clj b/test/metabase/query_processor_test/date_bucketing_test.clj
2 | index 60d8330db19..dbbc61b24ce 100644
3 | --- a/test/metabase/query_processor_test/date_bucketing_test.clj
4 | +++ b/test/metabase/query_processor_test/date_bucketing_test.clj
5 | @@ -1269,7 +1269,7 @@
6 | (testing "4 checkins per minute dataset"
7 | (testing "group by minute"
8 | (doseq [args [[:current] [-1 :minute] [1 :minute]]]
9 | - (is (= 4
10 | + (is (= 0
11 | (apply count-of-grouping checkins:4-per-minute :minute args))
12 | (format "filter by minute = %s" (into [:relative-datetime] args))))))))
13 |
14 |
--------------------------------------------------------------------------------
/src/metabase/driver/materialize.clj:
--------------------------------------------------------------------------------
1 | (ns metabase.driver.materialize
2 | "Metabase Materialize Driver."
3 | (:require [clojure
4 | [set :as set]]
5 | [honey.sql :as sql]
6 | [honey.sql.helpers :as sql.helpers]
7 | [metabase.db.spec :as db.spec]
8 | [metabase.config :as config]
9 | [metabase.driver :as driver]
10 | [metabase.util :as u]
11 | [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
12 | [metabase.driver.sql.query-processor :as sql.qp]
13 | [metabase.util.honey-sql-2 :as h2x]
14 | [metabase.driver.sync :as driver.s]
15 | [metabase.driver.sql-jdbc
16 | [common :as sql-jdbc.common]
17 | [connection :as sql-jdbc.conn]
18 | [sync :as sql-jdbc.sync]]))
19 |
20 | (driver/register! :materialize, :parent :postgres)
21 |
22 | (defmethod sql.qp/add-interval-honeysql-form :materialize
23 | [_driver hsql-form amount unit]
24 | ;; Convert weeks to days because Materialize doesn't support weeks and the rest should work as is
25 | (let [adjusted-amount (if (= unit :week) (* 7 amount) amount)
26 | adjusted-unit (if (= unit :week) :day unit)]
27 | (h2x// (sql.qp/add-interval-honeysql-form :postgres hsql-form adjusted-amount adjusted-unit))))
28 |
29 | ;;; +----------------------------------------------------------------------------------------------------------------+
30 | ;;; | metabase.driver method impls |
31 | ;;; +----------------------------------------------------------------------------------------------------------------+
32 |
33 | (doseq [[feature supported?] {:foreign-keys (not config/is-test?)
34 | :metadata/key-constraints (not config/is-test?)
35 | :foreign-keys-as-required-by-tests false
36 | ;; Materialize defaults to UTC, and this is the only supported value
37 | :set-timezone false
38 | :datetime-diff false
39 | :convert-timezone (not config/is-test?)
40 | :temporal-extract (not config/is-test?)
41 | ;; Disabling during tests as the data load fails with:
42 | ;; metabase.driver.sql-jdbc.sync.describe-table-test/describe-big-nested-field-columns-test (impl.clj:141)
43 | ;; ERROR: column "big_json" is of type jsonb but expression is of type character varying
44 | :nested-field-columns (not config/is-test?)
45 | ;; Disabling nested queries during tests as they try to use Foreign Keys
46 | :nested-queries (not config/is-test?)
47 | :regex false
48 | ;; Disabling model caching:
49 | :persist-models false
50 | ;; Disable percentile aggregations due to missing support for PERCENTILE_CONT
51 | :percentile-aggregations false
52 | ;; Disabling the support for the `:connection-impersonation` feature as it's not supported
53 | :connection-impersonation false
54 | ;; Disable uploads
55 | :uploads false
56 | :test/jvm-timezone-setting false}]
57 | (defmethod driver/database-supports? [:materialize feature] [_driver _feature _db] supported?))
58 |
59 | (defmethod sql-jdbc.execute/set-timezone-sql :materialize
60 | [_]
61 | "SET TIMEZONE TO %s;")
62 |
63 | ; ;;; +----------------------------------------------------------------------------------------------------------------+
64 | ; ;;; | metabase.driver.sql-jdbc impls |
65 | ; ;;; +----------------------------------------------------------------------------------------------------------------+
66 |
67 | (def ^:private default-materialize-connection-details
68 | {:host "materialize", :port 6875, :db "materialize", :cluster "quickstart"})
69 |
70 | (defn- validate-connection-details
71 | [{:keys [host]}]
72 | (when-not (re-matches #"^[a-zA-Z0-9.-]+$" host)
73 | (throw (IllegalArgumentException. (str "Invalid host: " host)))))
74 |
75 | (defmethod sql-jdbc.conn/connection-details->spec :materialize
76 | [_ details]
77 | (let [merged-details (merge default-materialize-connection-details details)
78 | ;; TODO: get the driver version from the plugin manifest instead of hardcoding it
79 | driver-version "v1.6.0"
80 | app-name (format "Metabase Materialize driver %s %s"
81 | driver-version
82 | config/mb-app-id-string)]
83 | (validate-connection-details merged-details)
84 | (let [{:keys [host port db cluster ssl], :as opts} merged-details]
85 | (sql-jdbc.common/handle-additional-options
86 | (merge
87 | {:classname "org.postgresql.Driver"
88 | :subprotocol "postgresql"
89 | :subname (str "//" host ":" port "/" db "?options=--cluster%3D" cluster)
90 | :sslmode (if ssl "require" "disable")
91 | :OpenSourceSubProtocolOverride false
92 | :ApplicationName app-name}
93 | (dissoc opts :host :port :db :cluster :ssl))))))
94 |
95 | (defmethod driver/describe-table :materialize
96 | [driver database table]
97 | (sql-jdbc.sync/describe-table driver database table))
98 |
99 | (defmethod sql-jdbc.sync/excluded-schemas :materialize [_driver] #{"mz_catalog" "mz_internal" "pg_catalog"})
100 |
101 | (defn ^:private get-tables-sql
102 | "Materialize doesn't support the pg_stat_user_tables table
103 | Overriding the default implementation to exclude the pg_stat_user_tables table"
104 | [schemas table-names]
105 | (sql/format
106 | (cond-> {:select [[:n.nspname :schema]
107 | [:c.relname :name]
108 | [[:case-expr :c.relkind
109 | [:inline "r"] [:inline "TABLE"]
110 | [:inline "p"] [:inline "PARTITIONED TABLE"]
111 | [:inline "v"] [:inline "VIEW"]
112 | [:inline "f"] [:inline "FOREIGN TABLE"]
113 | [:inline "m"] [:inline "MATERIALIZED VIEW"]
114 | :else nil]
115 | :type]
116 | [:d.description :description]]
117 | :from [[:pg_catalog.pg_class :c]]
118 | :join [[:pg_catalog.pg_namespace :n] [:= :c.relnamespace :n.oid]]
119 | :left-join [[:pg_catalog.pg_description :d] [:and [:= :c.oid :d.objoid]
120 | [:= :d.objsubid 0]
121 | [:= :d.classoid [:raw "'pg_class'::regclass"]]]]
122 | :where [:and [:= :c.relnamespace :n.oid]
123 | ;; filter out system tables (pg_ and mz_)
124 | [:and
125 | [(keyword "!~") :n.nspname "^pg_"]
126 | [(keyword "!~") :n.nspname "^mz_"]
127 | [:<> :n.nspname "information_schema"]]
128 | ;; only get tables of type: TABLE, PARTITIONED TABLE, VIEW, FOREIGN TABLE, MATERIALIZED VIEW
129 | [:raw "c.relkind in ('r', 'p', 'v', 'f', 'm')"]]
130 | :order-by [:type :schema :name]}
131 | (seq schemas)
132 | (sql.helpers/where [:in :n.nspname schemas])
133 |
134 | (seq table-names)
135 | (sql.helpers/where [:in :c.relname table-names]))
136 | {:dialect :ansi}))
137 |
138 | (defn- describe-database-tables
139 | [database]
140 | (let [[inclusion-patterns
141 | exclusion-patterns] (driver.s/db-details->schema-filter-patterns database)
142 | syncable? (fn [schema]
143 | (driver.s/include-schema? inclusion-patterns exclusion-patterns schema))]
144 | (eduction
145 | (comp (filter (comp syncable? :schema))
146 | (map #(dissoc % :type)))
147 | (sql-jdbc.execute/reducible-query database (get-tables-sql nil nil)))))
148 |
149 | (defmethod driver/describe-database :materialize
150 | [_driver database]
151 | ;; TODO: change this to return a reducible so we don't have to hold 100k tables in memory in a set like this
152 | {:tables (into #{} (describe-database-tables database))})
153 |
154 | ;; Overriding the default implementation to exclude the usage of the `format` function as it's not supported in Materialize
155 | (defmethod sql-jdbc.sync/describe-fields-sql :materialize
156 | [driver & {:keys [schema-names table-names]}]
157 | (sql/format
158 | {:select [[:c.column_name :name]
159 | [:c.data_type :database-type]
160 | [[:- :c.ordinal_position [:inline 1]] :database-position]
161 | [:c.table_schema :table-schema]
162 | [:c.table_name :table-name]
163 | [[:not= :pk.column_name nil] :pk?]
164 | ;; Materialize doesn't support column comments
165 | [nil :field-comment]
166 | ;; Materialize doesn't enforce NOT NULL constraints
167 | [false :database-required]
168 | ;; Materialize doesn't support auto-increment
169 | [false :database-is-auto-increment]]
170 | :from [[:information_schema.columns :c]]
171 | :left-join [[{:select [:tc.table_schema
172 | :tc.table_name
173 | :kc.column_name]
174 | :from [[:information_schema.table_constraints :tc]]
175 | :join [[:information_schema.key_column_usage :kc]
176 | [:and
177 | [:= :tc.constraint_name :kc.constraint_name]
178 | [:= :tc.table_schema :kc.table_schema]
179 | [:= :tc.table_name :kc.table_name]]]
180 | :where [:= :tc.constraint_type [:inline "PRIMARY KEY"]]}
181 | :pk]
182 | [:and
183 | [:= :c.table_schema :pk.table_schema]
184 | [:= :c.table_name :pk.table_name]
185 | [:= :c.column_name :pk.column_name]]]
186 | :where [:and
187 | [:raw "c.table_schema NOT IN ('mz_catalog', 'mz_internal', 'pg_catalog', 'information_schema')"]
188 | (when (seq schema-names)
189 | [:in :c.table_schema schema-names])
190 | (when (seq table-names)
191 | [:in :c.table_name table-names])]
192 | :order-by [[:c.table_schema :asc]
193 | [:c.table_name :asc]
194 | [:c.ordinal_position :asc]]}
195 | :dialect (sql.qp/quote-style driver)))
196 |
197 | (defmethod sql.qp/cast-temporal-string [:materialize :Coercion/YYYYMMDDHHMMSSString->Temporal]
198 | [_driver _coercion-strategy expr]
199 | [:make_timestamp
200 | [:cast [:substring expr (int 1) (int 4)] :integer]
201 | [:cast [:substring expr (int 5) (int 2)] :integer]
202 | [:cast [:substring expr (int 7) (int 2)] :integer]
203 | [:cast [:substring expr (int 9) (int 2)] :integer]
204 | [:cast [:substring expr (int 11) (int 2)] :integer]
205 | [:cast [:substring expr (int 13) (int 2)] :integer]])
206 |
--------------------------------------------------------------------------------
/test/metabase/driver/materialize_test.clj:
--------------------------------------------------------------------------------
1 | (ns metabase.driver.materialize-test
2 | (:require
3 | [clojure.test :refer :all]
4 | #_{:clj-kondo/ignore [:discouraged-namespace]}
5 | [honey.sql :as sql]
6 | [metabase.driver :as driver]
7 | [metabase.driver.materialize :as materialize]
8 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
9 | [metabase.driver.sql.query-processor :as sql.qp]
10 | [metabase.query-processor :as qp]
11 | [metabase.test.fixtures :as fixtures]
12 | [metabase.test :as mt]))
13 |
14 | (set! *warn-on-reflection* true)
15 |
16 | (use-fixtures :once (fixtures/initialize :plugins))
17 | (use-fixtures :once (fixtures/initialize :db))
18 |
--------------------------------------------------------------------------------
/test/metabase/test/data/materialize.clj:
--------------------------------------------------------------------------------
1 | (ns metabase.test.data.materialize
2 | "Test extensions for the Materialize driver. Includes logic for creating/destroying test datasets, building
3 | the connection specs from environment variables, etc."
4 | (:require
5 | [clojure.string :as str]
6 | [metabase.config :as config]
7 | [metabase.driver :as driver]
8 | [metabase.driver.ddl.interface :as ddl.i]
9 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
10 | [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
11 | [metabase.driver.sql-jdbc.sync.describe-table-test :as describe-table-test]
12 | [metabase.query-processor-test.alternative-date-test :as alternative-date-test]
13 | [metabase.query-processor-test.date-bucketing-test :as date-bucketing-test]
14 | [metabase.test.data.interface :as tx]
15 | [metabase.test.data.sql :as sql.tx]
16 | [metabase.test.data.sql-jdbc :as sql-jdbc.tx]
17 | [metabase.test.data.sql-jdbc.execute :as execute]
18 | [metabase.test.data.sql-jdbc.load-data :as load-data]
19 | [metabase.test.data.sql.ddl :as ddl]
20 | [metabase.util.log :as log]
21 | [metabase.util.malli :as mu]))
22 |
23 | (set! *warn-on-reflection* true)
24 |
25 | (defmethod ddl/drop-db-ddl-statements :materialize
26 | [& args]
27 | (apply (get-method ddl/drop-db-ddl-statements :sql-jdbc/test-extensions) args))
28 |
29 | (defmethod tx/aggregate-column-info :materialize
30 | ([driver ag-type]
31 | ((get-method tx/aggregate-column-info ::tx/test-extensions) driver ag-type))
32 |
33 | ([driver ag-type field]
34 | (cond-> ((get-method tx/aggregate-column-info ::tx/test-extensions) driver ag-type field)
35 | (= ag-type :sum) (assoc :base_type :type/BigInteger))))
36 |
37 | (doseq [[base-type db-type] {:type/BigInteger "BIGINT"
38 | :type/Boolean "BOOL"
39 | :type/Date "DATE"
40 | :type/DateTime "TIMESTAMP"
41 | :type/Decimal "DECIMAL"
42 | :type/Float "FLOAT"
43 | :type/Integer "INTEGER"
44 | :type/Text "TEXT"
45 | :type/JSON "JSON"
46 | :type/Time "TIME"
47 | :type/UUID "UUID"}]
48 | (defmethod sql.tx/field-base-type->sql-type [:materialize base-type] [_ _] db-type))
49 |
50 | (defmethod tx/dbdef->connection-details :materialize
51 | [_ context {:keys [database-name]}]
52 | (merge
53 | {:host (tx/db-test-env-var-or-throw :materialize :host "localhost")
54 | :ssl (tx/db-test-env-var :materialize :ssl false)
55 | :port (tx/db-test-env-var-or-throw :materialize :port 6877)
56 | :cluster (tx/db-test-env-var :materialize :cluster "quickstart")
57 | :user (tx/db-test-env-var-or-throw :materialize :user "mz_system")}
58 | (when-let [password (tx/db-test-env-var :materialize :password)]
59 | {:password password})
60 | (when (= context :db)
61 | {:db database-name})))
62 |
63 | (defmethod sql.tx/drop-table-if-exists-sql :materialize
64 | [driver {:keys [database-name]} {:keys [table-name]}]
65 | (format "DROP TABLE IF EXISTS \"%s\".\"%s\".\"%s\""
66 | (ddl.i/format-name driver database-name)
67 | "public"
68 | (ddl.i/format-name driver table-name)))
69 |
70 | (defmethod sql.tx/create-db-sql :materialize
71 | [driver {:keys [database-name]}]
72 | (format "CREATE DATABASE \"%s\";" (ddl.i/format-name driver database-name)))
73 | (defmethod sql.tx/drop-db-if-exists-sql :materialize
74 | [driver {:keys [database-name]}]
75 | (format "DROP DATABASE IF EXISTS \"%s\";" (ddl.i/format-name driver database-name)))
76 |
77 | (defmethod sql.tx/add-fk-sql :materialize [& _] nil)
78 |
79 | (defmethod execute/execute-sql! :materialize [& args]
80 | (apply execute/sequentially-execute-sql! args))
81 |
82 | (defmethod sql.tx/pk-sql-type :materialize [_] "INTEGER")
83 |
84 | (defmethod sql.tx/create-table-sql :materialize
85 | [driver dbdef tabledef]
86 | (let [tabledef (update tabledef :field-definitions (fn [field-defs]
87 | (for [field-def field-defs]
88 | (dissoc field-def :not-null?))))
89 | ;; strip out the PRIMARY KEY stuff from the CREATE TABLE statement
90 | sql ((get-method sql.tx/create-table-sql :sql/test-extensions) driver dbdef tabledef)]
91 | (str/replace sql #", PRIMARY KEY \([^)]+\)" "")))
92 |
93 | (defmethod load-data/row-xform :materialize
94 | [_driver _dbdef tabledef]
95 | (load-data/maybe-add-ids-xform tabledef))
96 |
97 | (defmethod tx/dataset-already-loaded? :materialize
98 | [driver dbdef]
99 | (let [tabledef (first (:table-definitions dbdef))
100 | schema-name "public"
101 | table-name (:table-name tabledef)]
102 | (sql-jdbc.execute/do-with-connection-with-options
103 | driver
104 | (sql-jdbc.conn/connection-details->spec driver (tx/dbdef->connection-details driver :db dbdef))
105 | {:write? false}
106 | (fn [^java.sql.Connection conn]
107 | (with-open [rset (.getTables (.getMetaData conn)
108 | nil ; catalog
109 | schema-name ; schema
110 | table-name ; table
111 | (into-array String ["TABLE"]))]
112 | (.next rset))))))
113 |
114 | (defmethod load-data/chunk-size :materialize
115 | [_driver _dbdef _tabledef]
116 | 400)
117 |
118 | (defmethod tx/sorts-nil-first? :materialize
119 | [_driver _base-type]
120 | false)
121 |
122 | (defmethod driver/database-supports? [:materialize :test/time-type]
123 | [_driver _feature _database]
124 | false)
125 |
126 | (defmethod driver/database-supports? [:materialize :test/timestamptz-type]
127 | [_driver _feature _database]
128 | false)
129 |
130 | (defmethod driver/database-supports? [:materialize ::describe-table-test/describe-materialized-view-fields]
131 | [_driver _feature _database]
132 | false)
133 |
134 | (defmethod driver/database-supports? [:materialize ::describe-table-test/describe-view-fields]
135 | [_driver _feature _database]
136 | false)
137 |
138 | (defmethod driver/database-supports? [:materialize :test/creates-db-on-connect]
139 | [_driver _feature _database]
140 | true)
141 |
142 | (defmethod driver/database-supports? [:materialize ::alternative-date-test/yyyymmddhhss-binary-timestamps]
143 | [_driver _feature _database]
144 | false)
145 |
146 | (defmethod alternative-date-test/yyyymmddhhmmss-binary-dates-expected-rows :materialize
147 | [_driver]
148 | [[1 "foo" #t "2019-04-21T16:43"]
149 | [2 "bar" #t "2020-04-21T16:43"]
150 | [3 "baz" #t "2021-04-21T16:43"]])
151 |
152 | (defmethod alternative-date-test/yyyymmddhhmmss-dates-expected-rows :materialize
153 | [_driver]
154 | [[1 "foo" #t "2019-04-21T16:43"]
155 | [2 "bar" #t "2020-04-21T16:43"]
156 | [3 "baz" #t "2021-04-21T16:43"]])
157 |
158 |
159 | (defmethod date-bucketing-test/group-by-default-test-expected-rows :materialize
160 | [_driver]
161 | [["2015-06-01T10:31:00Z" 1]
162 | ["2015-06-01T16:06:00Z" 1]
163 | ["2015-06-01T17:23:00Z" 1]
164 | ["2015-06-01T18:55:00Z" 1]
165 | ["2015-06-01T21:04:00Z" 1]
166 | ["2015-06-01T21:19:00Z" 1]
167 | ["2015-06-02T02:13:00Z" 1]
168 | ["2015-06-02T05:37:00Z" 1]
169 | ["2015-06-02T08:20:00Z" 1]
170 | ["2015-06-02T11:11:00Z" 1]])
171 |
172 | (defmethod date-bucketing-test/group-by-default-test-2-expected-rows :materialize
173 | [_driver]
174 | [["2015-06-01T10:31:00Z" 1]
175 | ["2015-06-01T16:06:00Z" 1]
176 | ["2015-06-01T17:23:00Z" 1]
177 | ["2015-06-01T18:55:00Z" 1]
178 | ["2015-06-01T21:04:00Z" 1]
179 | ["2015-06-01T21:19:00Z" 1]
180 | ["2015-06-02T02:13:00Z" 1]
181 | ["2015-06-02T05:37:00Z" 1]
182 | ["2015-06-02T08:20:00Z" 1]
183 | ["2015-06-02T11:11:00Z" 1]])
184 |
185 | (defmethod date-bucketing-test/group-by-week-database-timezone-override-test-expected-rows :materialize
186 | [_driver]
187 | [["2015-05-30T17:00:00-07:00" 46]
188 | ["2015-06-06T17:00:00-07:00" 47]
189 | ["2015-06-13T17:00:00-07:00" 40]
190 | ["2015-06-20T17:00:00-07:00" 60]
191 | ["2015-06-27T17:00:00-07:00" 7]])
192 |
--------------------------------------------------------------------------------