├── .docker ├── setup │ ├── requirements.txt │ ├── Dockerfile │ └── setup.py ├── clickhouse │ ├── cluster │ │ ├── server1_macros.xml │ │ ├── server2_macros.xml │ │ ├── server1_config.xml │ │ └── server2_config.xml │ ├── single_node_tls │ │ ├── Dockerfile │ │ ├── certificates │ │ │ ├── ca.key │ │ │ ├── client.key │ │ │ ├── server.key │ │ │ ├── client.crt │ │ │ ├── ca.crt │ │ │ └── server.crt │ │ ├── users.xml │ │ └── config.xml │ ├── users.xml │ └── single_node │ │ ├── users.xml │ │ └── config.xml └── nginx │ └── local.conf ├── .github ├── deps.edn ├── ISSUE_TEMPLATE │ ├── feature_request.md │ ├── question.md │ └── bug_report.md ├── pull_request_template.md └── workflows │ ├── check-jdbc-snapshot.yml │ └── check.yml ├── deps.edn ├── .build └── Dockerfile ├── .gitignore ├── AUTHORS.md ├── HISTORY.md ├── .static └── clickhouse.svg ├── resources └── metabase-plugin.yaml ├── src └── metabase │ └── driver │ ├── clickhouse_nippy.clj │ ├── clickhouse_version.clj │ ├── clickhouse_introspection.clj │ ├── clickhouse.clj │ └── clickhouse_qp.clj ├── README.md ├── CONTRIBUTING.md ├── docker-compose.yml ├── test └── metabase │ ├── driver │ ├── clickhouse_temporal_bucketing_test.clj │ ├── clickhouse_impersonation_test.clj │ ├── clickhouse_test.clj │ ├── clickhouse_substitution_test.clj │ └── clickhouse_introspection_test.clj │ └── test │ └── data │ ├── clickhouse_datasets.sql │ └── clickhouse.clj ├── LICENSE └── CHANGELOG.md /.docker/setup/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.32.2 2 | -------------------------------------------------------------------------------- /.docker/setup/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11.9-alpine 2 | COPY . /app/ 3 | RUN pip install -r /app/requirements.txt 4 | -------------------------------------------------------------------------------- /.github/deps.edn: -------------------------------------------------------------------------------- 1 | {:aliases 2 | {:user/clickhouse 3 | {:extra-paths ["PWD/modules/drivers/clickhouse/test"] 4 | :extra-deps {metabase/clickhouse {:local/root "PWD/modules/drivers/clickhouse"}}}}} 5 | -------------------------------------------------------------------------------- /.docker/clickhouse/cluster/server1_macros.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | test_cluster 4 | clickhouse1 5 | 1 6 | 7 | 8 | -------------------------------------------------------------------------------- /.docker/clickhouse/cluster/server2_macros.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | test_cluster 4 | clickhouse2 5 | 1 6 | 7 | 8 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "resources"] 2 | :deps 3 | {com.widdindustries/cljc.java-time {:mvn/version "0.1.21"} 4 | com.clickhouse/clickhouse-jdbc {:mvn/version "0.8.4"} 5 | org.lz4/lz4-java {:mvn/version "1.8.0"}}} 6 | -------------------------------------------------------------------------------- /.build/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG METABASE_VERSION 2 | FROM metabase/metabase:$METABASE_VERSION 3 | COPY ./clickhouse.metabase-driver.jar /plugins/clickhouse.metabase-driver.jar 4 | RUN chmod 744 /plugins/clickhouse.metabase-driver.jar 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | \#*\# 2 | .\#* 3 | /target 4 | *.jar 5 | *.class 6 | /.lein-env 7 | /.lein-failures 8 | /.lein-plugins 9 | /.lein-repl-history 10 | /.clj-kondo 11 | .eastwood 12 | .calva 13 | .cpcache 14 | .joyride 15 | .nrepl-port 16 | .idea 17 | -------------------------------------------------------------------------------- /.docker/nginx/local.conf: -------------------------------------------------------------------------------- 1 | upstream clickhouse_cluster { 2 | server clickhouse1:8123; 3 | server clickhouse2:8123; 4 | } 5 | 6 | server { 7 | listen 8123; 8 | client_max_body_size 100M; 9 | location / { 10 | proxy_pass http://clickhouse_cluster; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Contributors, Authors 2 | 3 | The initial source base comprises major contributions from these authors (_the git log has suffered from frequent brutal rebases, please add yourself here, if I missed you!_): 4 | 5 | * Bogdan Mukvich (@Badya) 6 | * @tsl-karlp 7 | * Andrew Grigorev (@ei-grad) 8 | * Felix Mueller (@enqueue) 9 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node_tls/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM clickhouse/clickhouse-server:25.2-alpine 2 | COPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs 3 | RUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \ 4 | && chmod 600 /etc/clickhouse-server/certs/* \ 5 | && chmod 755 /etc/clickhouse-server/certs 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for the driver 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | ### Use case 12 | 13 | ### Describe the solution you'd like 14 | 15 | ### Describe the alternatives you've considered 16 | 17 | ### Additional context 18 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node_tls/certificates/ca.key: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PARAMETERS----- 2 | BgUrgQQAIg== 3 | -----END EC PARAMETERS----- 4 | -----BEGIN EC PRIVATE KEY----- 5 | MIGkAgEBBDA26CfROCd9PBA7jUKmImXeYSuExnZcdNeX6AR0Aaj1wgZ/iH3hoQYH 6 | bPi7JnRTo+6gBwYFK4EEACKhZANiAATTKvPxkWILniWZ9EmcftQRqhH7fpVhQm1h 7 | vtZW1cpTozV0z6tdopnS5p/Wl+Kti2k/kZx1rsN1ZrRYKJN8ANruJJ6vaDOjbf89 8 | cmViZ/dbOi49T8brTzdHeuGIE2TyP+U= 9 | -----END EC PRIVATE KEY----- 10 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node_tls/certificates/client.key: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PARAMETERS----- 2 | BgUrgQQAIg== 3 | -----END EC PARAMETERS----- 4 | -----BEGIN EC PRIVATE KEY----- 5 | MIGkAgEBBDCsmtNREbSJvGTTIQmx019frOllv9m2EqOWesvqu52rH6tIxdw5TE4A 6 | SZICvNKYzH2gBwYFK4EEACKhZANiAAQa0kr/vcR7KfAYHt8HaDvt1XQzOAw66HBN 7 | BpvhFuuTDev+qA5m8ijgdSW02NGqt5SDfT7VImwqxg6nFarUJ4oZ70yAy6itF5cb 8 | 7+iXNLeg712GioYOoyIprQezRL8rBlc= 9 | -----END EC PRIVATE KEY----- 10 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node_tls/certificates/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PARAMETERS----- 2 | BgUrgQQAIg== 3 | -----END EC PARAMETERS----- 4 | -----BEGIN EC PRIVATE KEY----- 5 | MIGkAgEBBDDyhVuBtGuyKEDr2HciKi4yS2T0WloMeUG2kgQRKim7Mih7977q7RbI 6 | t6sGwlsBxKGgBwYFK4EEACKhZANiAAQKy8dFjE+v6Qn8DsMNqMS7w+F075YbHMmz 7 | uT1J3Kk9ZLE5oIMwTa1arS3QaLLrclg2Gyf4ImCU0heG19LmcdLs1XlJ8EAFRW1l 8 | Lb8wWTwLAULP5Fxnh1vaC33fsCCN/EI= 9 | -----END EC PRIVATE KEY----- 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Ask a question about the driver 4 | title: '' 5 | labels: question 6 | assignees: '' 7 | 8 | --- 9 | 10 | > Make sure to check the [documentation](https://clickhouse.com/docs/en/integrations/metabase) first. 11 | > If the question is concise and probably has a short answer, 12 | > asking it in the [community Slack](https://clickhouse.com/slack) is probably the fastest way to find the answer. 13 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Summary 2 | 3 | 4 | ## Checklist 5 | Delete items not relevant to your PR: 6 | - [ ] Unit and integration tests covering the common scenarios were added 7 | - [ ] A human-readable description of the changes was provided to include in CHANGELOG 8 | - [ ] For significant changes, documentation in https://github.com/ClickHouse/clickhouse-docs was updated with further explanations or tutorials 9 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | # History 2 | The request for a ClickHouse Metabase driver is formulated in [Metabase issue #3332](https://github.com/metabase/metabase/issues/3332). Some impatient ClickHouse users started development. The Metabase team is asking driver developers to publish plug-ins and collect some experiences before considering a PR, so here we are. 3 | 4 | This driver is based on the following PRs: 5 | * [metabase#8491](https://github.com/metabase/metabase/pull/8491) 6 | * [metabase#8722](https://github.com/metabase/metabase/pull/8722) 7 | * [metabase#9469](https://github.com/metabase/metabase/pull/9469) 8 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node_tls/certificates/client.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIB+TCCAX8CFEc86vC0vsMjLzQzxazHeHjQblL2MAoGCCqGSM49BAMEMF0xCzAJ 3 | BgNVBAYTAlVTMQswCQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25u 4 | ZWN0IFRlc3QxHzAdBgNVBAMMFmNsaWNraG91c2Vjb25uZWN0LnRlc3QwHhcNMjIw 5 | NTE5MjEwNTA2WhcNNDIwNTEzMjEwNTA2WjBkMQswCQYDVQQGEwJVUzELMAkGA1UE 6 | CAwCQ0ExIDAeBgNVBAoMF0NsaWNrSG91c2UgQ29ubmVjdCBUZXN0MSYwJAYDVQQD 7 | DB1jbGllbnQuY2xpY2tob3VzZWNvbm5lY3QudGVzdDB2MBAGByqGSM49AgEGBSuB 8 | BAAiA2IABBrSSv+9xHsp8Bge3wdoO+3VdDM4DDrocE0Gm+EW65MN6/6oDmbyKOB1 9 | JbTY0aq3lIN9PtUibCrGDqcVqtQnihnvTIDLqK0Xlxvv6Jc0t6DvXYaKhg6jIimt 10 | B7NEvysGVzAKBggqhkjOPQQDBANoADBlAjBblevbpaRlekX7fH16KnYttGoIqDBI 11 | 45LlBJ2sEe5qSKCBoLdN89Tk8WD4lG7PhlkCMQDdFd8OKMPaZiUWIdHI6AeDWwXD 12 | bJi0LwDxXgyBVCGLZ2vTbOVxnr2Qp+9BjFURU8c= 13 | -----END CERTIFICATE----- 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | ### Describe the bug 12 | 13 | ### Steps to reproduce 14 | 1. 15 | 2. 16 | 3. 17 | 18 | ### Expected behaviour 19 | 20 | ### Error log 21 | 22 | ### Configuration 23 | #### Environment 24 | * metabase-clickhouse-driver version: 25 | * metabase-clickhouse-driver configuration: 26 | * Metabase version: 27 | * OS: 28 | 29 | #### ClickHouse server 30 | * ClickHouse Server version: 31 | * ClickHouse Server non-default settings, if any: 32 | * `CREATE TABLE` statements for tables involved: 33 | * Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary 34 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node_tls/certificates/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICTTCCAdKgAwIBAgIUaqbLNiwUtbV5VuolTMGXOO+21vEwCgYIKoZIzj0EAwQw 3 | XTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSAwHgYDVQQKDBdDbGlja0hvdXNl 4 | IENvbm5lY3QgVGVzdDEfMB0GA1UEAwwWY2xpY2tob3VzZWNvbm5lY3QudGVzdDAe 5 | Fw0yMjA1MTkxODIxMzFaFw00MjA1MTQxODIxMzFaMF0xCzAJBgNVBAYTAlVTMQsw 6 | CQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25uZWN0IFRlc3QxHzAd 7 | BgNVBAMMFmNsaWNraG91c2Vjb25uZWN0LnRlc3QwdjAQBgcqhkjOPQIBBgUrgQQA 8 | IgNiAATTKvPxkWILniWZ9EmcftQRqhH7fpVhQm1hvtZW1cpTozV0z6tdopnS5p/W 9 | l+Kti2k/kZx1rsN1ZrRYKJN8ANruJJ6vaDOjbf89cmViZ/dbOi49T8brTzdHeuGI 10 | E2TyP+WjUzBRMB0GA1UdDgQWBBThZgdf9aToyK2TeSQ+suyjNUuifDAfBgNVHSME 11 | GDAWgBThZgdf9aToyK2TeSQ+suyjNUuifDAPBgNVHRMBAf8EBTADAQH/MAoGCCqG 12 | SM49BAMEA2kAMGYCMQDWQUTb39xLLds0WobJmNQbIkEwZyss0XNQkn6qI8rz73NL 13 | 6L5/6wNzetKhBf3WBCYCMQC+evVR3Td+WLfbKQDXrCbSkogW6++I/9l55wakMz9G 14 | P+0she/nvFuUKnB+VRcaBqM= 15 | -----END CERTIFICATE----- 16 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node_tls/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | random 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | ::/0 15 | 16 | default 17 | default 18 | 1 19 | 20 | 21 | 22 | client.clickhouseconnect.test 23 | 24 | default 25 | 26 | 27 | 28 | 29 | 30 | 31 | 3600 32 | 0 33 | 0 34 | 0 35 | 0 36 | 0 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /.docker/clickhouse/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | random 7 | 1 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | ::/0 16 | 17 | default 18 | default 19 | 1 20 | 21 | 22 | foo@bar! 23 | 24 | ::/0 25 | 26 | default 27 | default 28 | 1 29 | 30 | 31 | 32 | 33 | 34 | 35 | 3600 36 | 0 37 | 0 38 | 0 39 | 0 40 | 0 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | random 7 | 1 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | ::/0 16 | 17 | default 18 | default 19 | 1 20 | 21 | 22 | foo@bar! 23 | 24 | ::/0 25 | 26 | default 27 | default 28 | 1 29 | 30 | 31 | 32 | 33 | 34 | 35 | 3600 36 | 0 37 | 0 38 | 0 39 | 0 40 | 0 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node_tls/certificates/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDPTCCAsOgAwIBAgIURzzq8LS+wyMvNDPFrMd4eNBuUvUwCgYIKoZIzj0EAwQw 3 | XTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSAwHgYDVQQKDBdDbGlja0hvdXNl 4 | IENvbm5lY3QgVGVzdDEfMB0GA1UEAwwWY2xpY2tob3VzZWNvbm5lY3QudGVzdDAe 5 | Fw0yMjA1MTkyMDU3MjRaFw00MjA1MTMyMDU3MjRaMGQxCzAJBgNVBAYTAlVTMQsw 6 | CQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25uZWN0IFRlc3QxJjAk 7 | BgNVBAMMHXNlcnZlci5jbGlja2hvdXNlY29ubmVjdC50ZXN0MHYwEAYHKoZIzj0C 8 | AQYFK4EEACIDYgAECsvHRYxPr+kJ/A7DDajEu8PhdO+WGxzJs7k9SdypPWSxOaCD 9 | ME2tWq0t0Giy63JYNhsn+CJglNIXhtfS5nHS7NV5SfBABUVtZS2/MFk8CwFCz+Rc 10 | Z4db2gt937AgjfxCo4IBOzCCATcwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMC 11 | BkAwOQYJYIZIAYb4QgENBCwWKkNsaWNrSG91c2UgQ29ubmVjdCBUZXN0IFNlcnZl 12 | ciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUZDd2tpXw4FMDFcY38eXCb+tmukAwgZoG 13 | A1UdIwSBkjCBj4AU4WYHX/Wk6Mitk3kkPrLsozVLonyhYaRfMF0xCzAJBgNVBAYT 14 | AlVTMQswCQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25uZWN0IFRl 15 | c3QxHzAdBgNVBAMMFmNsaWNraG91c2Vjb25uZWN0LnRlc3SCFGqmyzYsFLW1eVbq 16 | JUzBlzjvttbxMAsGA1UdDwQEAwIF4DATBgNVHSUEDDAKBggrBgEFBQcDATAKBggq 17 | hkjOPQQDBANoADBlAjBc3W/8qr04xmUiDOHSEoug89cK8YxtRiKdCjiR3Lao1h5a 18 | J5Xc0JhVLaDUFb+blkoCMQCM7rKbO3itBKaweeJijX/veBcISYFulryWeANiltxo 19 | DFDHrC54rGXt4eOMouTlPbw= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8123 5 | 9000 6 | 7 | users.xml 8 | default 9 | default 10 | 11 | 5368709120 12 | 13 | /var/lib/clickhouse/ 14 | /var/lib/clickhouse/tmp/ 15 | /var/lib/clickhouse/user_files/ 16 | /var/lib/clickhouse/access/ 17 | UTC 18 | 19 | 20 | debug 21 | /var/log/clickhouse-server/clickhouse-server.log 22 | /var/log/clickhouse-server/clickhouse-server.err.log 23 | 1000M 24 | 10 25 | 1 26 | 27 | 28 | 29 | system 30 | query_log
31 | toYYYYMM(event_date) 32 | 1000 33 | 34 | 35 | 36 | /var/lib/clickhouse/format_schemas/ 37 | 38 | 39 | users.xml 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /.docker/clickhouse/single_node_tls/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8443 5 | 9440 6 | 0.0.0.0 7 | 8 | users.xml 9 | default 10 | default 11 | 12 | 5368709120 13 | 14 | /var/lib/clickhouse/ 15 | /var/lib/clickhouse/tmp/ 16 | /var/lib/clickhouse/user_files/ 17 | /var/lib/clickhouse/access/ 18 | 19 | 20 | debug 21 | /var/log/clickhouse-server/clickhouse-server.log 22 | /var/log/clickhouse-server/clickhouse-server.err.log 23 | 1000M 24 | 10 25 | 1 26 | 27 | 28 | 29 | 30 | /etc/clickhouse-server/certs/server.crt 31 | /etc/clickhouse-server/certs/server.key 32 | relaxed 33 | /etc/clickhouse-server/certs/ca.crt 34 | true 35 | sslv2,sslv3,tlsv1 36 | true 37 | 38 | 39 | 40 | 41 | system 42 | query_log
43 | toYYYYMM(event_date) 44 | 1000 45 | 46 | 47 | 48 | /var/lib/clickhouse/format_schemas/ 49 | 50 | 51 | users.xml 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /.static/clickhouse.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /resources/metabase-plugin.yaml: -------------------------------------------------------------------------------- 1 | info: 2 | name: Metabase ClickHouse Driver 3 | version: 1.53.4 4 | description: Allows Metabase to connect to ClickHouse databases. 5 | contact-info: 6 | name: ClickHouse 7 | address: https://github.com/ClickHouse/metabase-clickhouse-driver 8 | driver: 9 | name: clickhouse 10 | display-name: ClickHouse 11 | lazy-load: true 12 | parent: sql-jdbc 13 | connection-properties: 14 | - host 15 | - merge: 16 | - port 17 | - default: 8123 18 | - user 19 | - password 20 | - name: dbname 21 | display-name: Databases 22 | placeholder: default 23 | helper-text: "To specify multiple databases, separate them by the space symbol. For example: default data logs." 24 | - name: scan-all-databases 25 | display-name: Scan all databases 26 | type: boolean 27 | default: false 28 | description: Scan all tables from all available ClickHouse databases except the system ones. 29 | 30 | - ssl 31 | - ssh-tunnel 32 | - advanced-options-start 33 | - name: use-no-proxy 34 | display-name: Disable system wide proxy settings 35 | default: false 36 | type: boolean 37 | visible-if: 38 | advanced-options: true 39 | - name: clickhouse-settings 40 | display-name: ClickHouse settings (comma-separated) 41 | placeholder: "allow_experimental_analyzer=1,max_result_rows=100" 42 | visible-if: 43 | advanced-options: true 44 | - name: max-open-connections 45 | display-name: "Max open HTTP connections in the JDBC driver (default: 100)" 46 | placeholder: 100 47 | visible-if: 48 | advanced-options: true 49 | - merge: 50 | - additional-options 51 | - placeholder: "connection_timeout=1000&socket_timeout=300000" 52 | - default-advanced-options 53 | connection-properties-include-tunnel-config: true 54 | init: 55 | - step: load-namespace 56 | namespace: metabase.driver.clickhouse 57 | - step: register-jdbc-driver 58 | class: com.clickhouse.jdbc.ClickHouseDriver 59 | -------------------------------------------------------------------------------- /src/metabase/driver/clickhouse_nippy.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.clickhouse-nippy 2 | (:require [taoensso.nippy :as nippy]) 3 | (:import [java.io DataInput DataOutput])) 4 | 5 | (set! *warn-on-reflection* false) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; com.clickhouse.data.value.UnsignedByte 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | (nippy/extend-freeze com.clickhouse.data.value.UnsignedByte :clickhouse/UnsignedByte 11 | [^com.clickhouse.data.value.UnsignedByte x ^DataOutput data-output] 12 | ;; can't enable *warn-on-reflection* because of this call 13 | (nippy/freeze-to-out! data-output (.toString x))) 14 | 15 | (nippy/extend-thaw :clickhouse/UnsignedByte 16 | [^DataInput data-input] 17 | (com.clickhouse.data.value.UnsignedByte/valueOf (nippy/thaw-from-in! data-input))) 18 | 19 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 20 | ;; com.clickhouse.data.value.UnsignedShort 21 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 22 | (nippy/extend-freeze com.clickhouse.data.value.UnsignedShort :clickhouse/UnsignedShort 23 | [^com.clickhouse.data.value.UnsignedShort x ^DataOutput data-output] 24 | (nippy/freeze-to-out! data-output (.toString x))) 25 | 26 | (nippy/extend-thaw :clickhouse/UnsignedShort 27 | [^DataInput data-input] 28 | (com.clickhouse.data.value.UnsignedShort/valueOf (nippy/thaw-from-in! data-input))) 29 | 30 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 31 | ;; com.clickhouse.data.value.UnsignedInteger 32 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 33 | (nippy/extend-freeze com.clickhouse.data.value.UnsignedInteger :clickhouse/UnsignedInteger 34 | [^com.clickhouse.data.value.UnsignedInteger x ^DataOutput data-output] 35 | (nippy/freeze-to-out! data-output (.toString x))) 36 | 37 | (nippy/extend-thaw :clickhouse/UnsignedInteger 38 | [^DataInput data-input] 39 | (com.clickhouse.data.value.UnsignedInteger/valueOf (nippy/thaw-from-in! data-input))) 40 | 41 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 42 | ;; com.clickhouse.data.value.UnsignedLong 43 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 44 | (nippy/extend-freeze com.clickhouse.data.value.UnsignedLong :clickhouse/UnsignedLong 45 | [^com.clickhouse.data.value.UnsignedLong x ^DataOutput data-output] 46 | (nippy/freeze-to-out! data-output (.toString x))) 47 | 48 | (nippy/extend-thaw :clickhouse/UnsignedLong 49 | [^DataInput data-input] 50 | (com.clickhouse.data.value.UnsignedLong/valueOf (nippy/thaw-from-in! data-input))) 51 | -------------------------------------------------------------------------------- /src/metabase/driver/clickhouse_version.clj: -------------------------------------------------------------------------------- 1 | "Provides the info about the ClickHouse version. Extracted from the main clickhouse.clj file, 2 | as both Driver and QP overrides require access to it, avoiding circular dependencies." 3 | (ns metabase.driver.clickhouse-version 4 | (:require [clojure.core.memoize :as memoize] 5 | [metabase.driver :as driver] 6 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] 7 | [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] 8 | [metabase.driver.util :as driver.u] 9 | [metabase.lib.metadata :as lib.metadata] 10 | [metabase.query-processor.store :as qp.store])) 11 | 12 | (set! *warn-on-reflection* true) 13 | 14 | ;; cache the results for 60 minutes; 15 | ;; TTL is here only to eventually clear out old entries/keep it from growing too large 16 | (def ^:private default-cache-ttl (* 60 60 1000)) 17 | 18 | (def ^:private clickhouse-version-query 19 | (str "WITH s AS (SELECT version() AS ver, splitByChar('.', ver) AS verSplit) " 20 | "SELECT s.ver, toInt32(verSplit[1]), toInt32(verSplit[2]) FROM s")) 21 | 22 | (def ^:private ^{:arglists '([db-details])} get-clickhouse-version 23 | (memoize/ttl 24 | (fn [db-details] 25 | (sql-jdbc.execute/do-with-connection-with-options 26 | :clickhouse 27 | (sql-jdbc.conn/connection-details->spec :clickhouse db-details) 28 | nil 29 | (fn [^java.sql.Connection conn] 30 | (with-open [stmt (.createStatement conn) 31 | rset (.executeQuery stmt clickhouse-version-query)] 32 | (when (.next rset) 33 | {:version (.getString rset 1) 34 | :semantic-version {:major (.getInt rset 2) 35 | :minor (.getInt rset 3)}}))))) 36 | :ttl/threshold default-cache-ttl)) 37 | 38 | (defmethod driver/dbms-version :clickhouse 39 | [_driver db] 40 | (get-clickhouse-version (:details db))) 41 | 42 | (defn is-at-least? 43 | "Is ClickHouse version at least `major.minor` (e.g., 24.4)?" 44 | ([major minor] 45 | ;; used from the QP overrides; we don't have access to the DB object 46 | (is-at-least? major minor (lib.metadata/database (qp.store/metadata-provider)))) 47 | ([major minor db] 48 | ;; used from the Driver overrides; we have access to the DB object 49 | (let [version (driver/dbms-version :clickhouse db) 50 | semantic (:semantic-version version)] 51 | (driver.u/semantic-version-gte [(:major semantic) (:minor semantic)] [major minor])))) 52 | 53 | (defn with-min 54 | "Execute `f` if the ClickHouse version is greater or equal to `major.minor` (e.g., 24.4); 55 | otherwise, execute `fallback-f`, if it's provided." 56 | ([major minor f] 57 | (with-min major minor f nil)) 58 | ([major minor f fallback-f] 59 | (if (is-at-least? major minor) 60 | (f) 61 | (when (not (nil? fallback-f)) (fallback-f))))) 62 | -------------------------------------------------------------------------------- /.github/workflows/check-jdbc-snapshot.yml: -------------------------------------------------------------------------------- 1 | # sed -i -E "s/(com.clickhouse\/clickhouse-jdbc \{:mvn\/version )\".+?\"\}/\1\"\"}/" deps.edn 2 | name: Check with the latest JDBC snapshot 3 | 4 | on: 5 | workflow_dispatch: 6 | inputs: 7 | jdbc-version: 8 | description: "JDBC version to use" 9 | required: true 10 | 11 | env: 12 | # Temporarily using a fork to disable a few failing tests 13 | METABASE_REPOSITORY: slvrtrn/metabase 14 | METABASE_VERSION: 0.53.x-ch 15 | 16 | jobs: 17 | check-local-current-version: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Checkout Metabase Repo 21 | uses: actions/checkout@v4 22 | with: 23 | repository: ${{ env.METABASE_REPOSITORY }} 24 | ref: ${{ env.METABASE_VERSION }} 25 | 26 | - name: Checkout Driver Repo 27 | uses: actions/checkout@v4 28 | with: 29 | path: modules/drivers/clickhouse 30 | 31 | - name: Prepare JDK 21 32 | uses: actions/setup-java@v4 33 | with: 34 | distribution: "temurin" 35 | java-version: "21" 36 | 37 | - name: Add ClickHouse TLS instance to /etc/hosts 38 | run: | 39 | sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts 40 | 41 | - name: Start ClickHouse in Docker 42 | uses: hoverkraft-tech/compose-action@v2.0.0 43 | with: 44 | compose-file: "modules/drivers/clickhouse/docker-compose.yml" 45 | down-flags: "--volumes" 46 | services: | 47 | clickhouse 48 | clickhouse_tls 49 | clickhouse_older_version 50 | clickhouse_cluster_node1 51 | clickhouse_cluster_node2 52 | nginx 53 | 54 | - name: Install Clojure CLI 55 | run: | 56 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh && 57 | sudo bash ./linux-install-1.11.1.1182.sh 58 | 59 | - name: Setup Node 60 | uses: actions/setup-node@v4 61 | with: 62 | node-version: "20" 63 | cache: "yarn" 64 | 65 | - name: Update the JDBC version in deps.edn 66 | working-directory: modules/drivers/clickhouse 67 | run: | 68 | sed -i -E "s/(com.clickhouse\/clickhouse-jdbc \{:mvn\/version )\".+?\"\}/\1\"${{ github.event.inputs.jdbc-version }}\"}/" deps.edn 69 | cat deps.edn 70 | 71 | - name: Prepare stuff for pulses 72 | run: yarn build-static-viz 73 | 74 | # Use custom deps.edn containing "user/clickhouse" alias to include driver sources 75 | - name: Prepare deps.edn 76 | run: | 77 | mkdir -p /home/runner/.config/clojure 78 | cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn 79 | 80 | - name: Run all tests with the latest ClickHouse version 81 | env: 82 | DRIVERS: clickhouse 83 | run: | 84 | clojure -X:ci:dev:ee:ee-dev:drivers:drivers-dev:test:user/clickhouse 85 | -------------------------------------------------------------------------------- /.docker/clickhouse/cluster/server1_config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8123 5 | 9009 6 | clickhouse1 7 | 8 | users.xml 9 | default 10 | default 11 | 12 | 5368709120 13 | 14 | /var/lib/clickhouse/ 15 | /var/lib/clickhouse/tmp/ 16 | /var/lib/clickhouse/user_files/ 17 | /var/lib/clickhouse/access/ 18 | 3 19 | 20 | 21 | debug 22 | /var/log/clickhouse-server/clickhouse-server.log 23 | /var/log/clickhouse-server/clickhouse-server.err.log 24 | 1000M 25 | 10 26 | 1 27 | 28 | 29 | 30 | 31 | 32 | 33 | clickhouse1 34 | 9000 35 | 36 | 37 | clickhouse2 38 | 9000 39 | 40 | 41 | 42 | 43 | 44 | 45 | 9181 46 | 1 47 | /var/lib/clickhouse/coordination/log 48 | /var/lib/clickhouse/coordination/snapshots 49 | 50 | 51 | 10000 52 | 30000 53 | trace 54 | 10000 55 | 56 | 57 | 58 | 59 | 1 60 | clickhouse1 61 | 9000 62 | 63 | 64 | 2 65 | clickhouse2 66 | 9000 67 | 68 | 69 | 70 | 71 | 72 | 73 | clickhouse1 74 | 9181 75 | 76 | 77 | clickhouse2 78 | 9181 79 | 80 | 81 | 82 | 83 | /clickhouse/test_cluster/task_queue/ddl 84 | 85 | 86 | 87 | system 88 | query_log
89 | toYYYYMM(event_date) 90 | 1000 91 | 92 | 93 | 94 | /var/lib/clickhouse/format_schemas/ 95 | 96 | 97 | users.xml 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /.docker/clickhouse/cluster/server2_config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8123 5 | 9009 6 | clickhouse2 7 | 8 | users.xml 9 | default 10 | default 11 | 12 | 5368709120 13 | 14 | /var/lib/clickhouse/ 15 | /var/lib/clickhouse/tmp/ 16 | /var/lib/clickhouse/user_files/ 17 | /var/lib/clickhouse/access/ 18 | 3 19 | 20 | 21 | debug 22 | /var/log/clickhouse-server/clickhouse-server.log 23 | /var/log/clickhouse-server/clickhouse-server.err.log 24 | 1000M 25 | 10 26 | 1 27 | 28 | 29 | 30 | 31 | 32 | 33 | clickhouse1 34 | 9000 35 | 36 | 37 | clickhouse2 38 | 9000 39 | 40 | 41 | 42 | 43 | 44 | 45 | 9181 46 | 2 47 | /var/lib/clickhouse/coordination/log 48 | /var/lib/clickhouse/coordination/snapshots 49 | 50 | 51 | 10000 52 | 30000 53 | trace 54 | 10000 55 | 56 | 57 | 58 | 59 | 1 60 | clickhouse1 61 | 9000 62 | 63 | 64 | 2 65 | clickhouse2 66 | 9000 67 | 68 | 69 | 70 | 71 | 72 | 73 | clickhouse1 74 | 9181 75 | 76 | 77 | clickhouse2 78 | 9181 79 | 80 | 81 | 82 | 83 | /clickhouse/test_cluster/task_queue/ddl 84 | 85 | 86 | 87 | system 88 | query_log
89 | toYYYYMM(event_date) 90 | 1000 91 | 92 | 93 | 94 | /var/lib/clickhouse/format_schemas/ 95 | 96 | 97 | users.xml 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | 4 |

ClickHouse driver for Metabase

5 |

6 |
7 |

8 | 9 | 10 | 11 |

12 | 13 | ## About 14 | 15 | The Metabase team promoted the ClickHouse driver to the core level as of Metabase 54 ([release notes](https://github.com/metabase/metabase/releases/tag/v0.54.1), [driver source](https://github.com/metabase/metabase/tree/v0.54.x/modules/drivers/clickhouse)). 16 | 17 | For the end user, this means the following: 18 | 19 | - Installing the driver manually is unnecessary, as it is now bundled with Metabase. 20 | - Starting from April 2025, the Metabase team will continue maintaining the driver. Please report new issues in [the main Metabase repository](https://github.com/metabase/metabase/issues). 21 | 22 | The ClickHouse team recommends avoiding older Metabase versions (53.x and earlier) with manual driver installation; instead, please use the updated Metabase distribution with the driver built-in. 23 | 24 | ## History 25 | 26 | The request for a ClickHouse Metabase driver was formulated in 2016 in [Metabase issue #3332](https://github.com/metabase/metabase/issues/3332). Several impatient ClickHouse users started the development in the main Metabase repo. In March 2019, after releasing the plugin SDK, the Metabase team [asked to publish the driver separately](https://github.com/metabase/metabase/pull/8491#issuecomment-471721980) in its own repository, and later that month, with Felix Mueller ([@enqueue](https://github.com/enqueue)) leading the efforts, the [initial version of the driver](https://github.com/ClickHouse/metabase-clickhouse-driver/releases/tag/v0.1) was out. 27 | 28 | The original implementation of the driver was based on the following pull requests: 29 | 30 | - [metabase#8491](https://github.com/metabase/metabase/pull/8491) 31 | - [metabase#8722](https://github.com/metabase/metabase/pull/8722) 32 | - [metabase#9469](https://github.com/metabase/metabase/pull/9469) 33 | 34 | The source base in these PRs comprises major contributions from these authors: 35 | 36 | - [@tsl-karlp](https://github.com/tsl-karlp) 37 | - Andrew Grigorev ([@ei-grad](https://github.com/ei-grad)) 38 | - Bogdan Mukvich ([@Badya](https://github.com/Badya)) 39 | - Felix Mueller ([@enqueue](https://github.com/enqueue)) 40 | 41 | > [!NOTE] 42 | > Special thanks to Felix Mueller ([@enqueue](https://github.com/enqueue)), who was the sole maintainer of the project from 2019 to 2022 before transferring it to ClickHouse. 43 | 44 | Starting from November 2022, Serge Klochkov ([@slvrtrn](https://github.com/slvrtrn)) joined as a maintainer. 45 | 46 | In early 2023, the repository was transferred to the ClickHouse organization, promoting it as an [official integration](https://clickhouse.com/blog/metabase-clickhouse-plugin-ga-release). Around that time, the driver also became available in [Metabase Cloud](https://www.metabase.com/cloud). 47 | 48 | In April 2025, the driver source code [was moved](https://github.com/metabase/metabase/pull/54740) to the main Metabase repository. Since [Metabase 54](https://github.com/metabase/metabase/releases/tag/v0.54.1), it is now available as a part of the official Metabase bundle. 49 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Getting started 2 | 3 | ClickHouse driver for Metabase is an open-source project, 4 | and we welcome any contributions from the community. 5 | Please share your ideas, contribute to the codebase, 6 | and help us maintain up-to-date documentation. 7 | 8 | * Please report any issues you encounter during operations. 9 | * Feel free to create a pull request, preferably with a test or five. 10 | 11 | ## Setting up a development environment 12 | 13 | ### Requirements 14 | 15 | * Clojure 1.11+ 16 | * OpenJDK 17 17 | * Node.js 16.x 18 | * Yarn 19 | 20 | For testing: Docker Compose 21 | 22 | Please refer to the extensive documentation available on the Metabase website: [Guide to writing a Metabase driver](https://www.metabase.com/docs/latest/developers-guide/drivers/start.html) 23 | 24 | ClickHouse driver's code should be inside the main Metabase repository checkout in `modules/drivers/clickhouse` directory. 25 | 26 | Additionally, you need to tweak Metabase's `deps.edn` a bit. 27 | 28 | The easiest way to set up a development environment is as follows (mostly the same as in the [CI](https://github.com/enqueue/metabase-clickhouse-driver/blob/master/.github/workflows/check.yml)): 29 | 30 | * Clone Metabase and ClickHouse driver repositories 31 | ```bash 32 | git clone https://github.com/metabase/metabase.git 33 | cd metabase 34 | git clone https://github.com/enqueue/metabase-clickhouse-driver.git modules/drivers/clickhouse 35 | ``` 36 | 37 | * Create custom Clojure profiles 38 | 39 | ```bash 40 | mkdir -p ~/.clojure 41 | cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > ~/.clojure/deps.edn 42 | ``` 43 | 44 | Modifying `~/.clojure/deps.edn` will create a new profile: `user/clickhouse`, that adds driver's sources to the class path, and includes all the Metabase tests that are guaranteed to work with the driver. 45 | 46 | * Install the Metabase dependencies: 47 | 48 | ```bash 49 | clojure -X:deps:drivers prep 50 | ``` 51 | 52 | * Build the frontend: 53 | 54 | ```bash 55 | yarn && yarn build-static-viz 56 | ``` 57 | 58 | * Add /etc/hosts entry 59 | 60 | Required for TLS tests. 61 | 62 | ```bash 63 | sudo -- sh -c "echo 127.0.0.1 server.clickhouseconnect.test >> /etc/hosts" 64 | ``` 65 | 66 | * Start Docker containers 67 | 68 | ```bash 69 | docker compose -f modules/drivers/clickhouse/docker-compose.yml up -d 70 | ``` 71 | 72 | Here's an overview of the started containers, which have the ports exposed to the `localhost` (see [docker-compose.yml](./docker-compose.yml)): 73 | 74 | - Metabase with the ClickHouse driver loaded from the JAR file (port: 3000) 75 | - Current ClickHouse version (port: 8123) - the main instance for all tests. 76 | - Current ClickHouse cluster with two nodes (+ nginx as an LB, port: 8127) - required for the set role tests (verifying that the role is set correctly via the query parameters). 77 | - Current ClickHouse version with TLS support (port: 8443) - required for the TLS tests. 78 | - Older ClickHouse version (port: 8124) - required for the string functions tests (switch between UTF8 (current) and non-UTF8 (pre-23.8) versions), as well as to verify that certain features, such as connection impersonation, are disabled on the older server versions. 79 | 80 | Now, you should be able to run the tests: 81 | 82 | ```bash 83 | DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse 84 | ``` 85 | 86 | you can see that we have our `:user/clickhouse` profile added to the command above, and with `DRIVERS=clickhouse` we instruct Metabase to run the tests only for ClickHouse. 87 | 88 | NB: Omitting `DRIVERS` will run the tests for all the built-in database drivers. 89 | 90 | If you want to run tests for only a specific namespace: 91 | 92 | ```bash 93 | DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test 94 | ``` 95 | 96 | or even a single test: 97 | 98 | ```bash 99 | DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test/clickhouse-nullable-arrays 100 | ``` 101 | 102 | Testing the driver with the older ClickHouse version (see [docker-compose.yml](./docker-compose.yml)): 103 | 104 | ```bash 105 | MB_CLICKHOUSE_TEST_PORT=8124 DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test 106 | ``` 107 | 108 | ## Building a jar 109 | 110 | You need to add an entry for ClickHouse in `modules/drivers/deps.edn` 111 | 112 | ```clj 113 | {:deps 114 | {... 115 | metabase/clickhouse {:local/root "clickhouse"} 116 | ...}} 117 | ``` 118 | 119 | or just run this from the root Metabase directory, overwriting the entire file: 120 | 121 | ```bash 122 | echo "{:deps {metabase/clickhouse {:local/root \"clickhouse\" }}}" > modules/drivers/deps.edn 123 | ``` 124 | 125 | Now, you should be able to build the final jar: 126 | 127 | ```bash 128 | bin/build-driver.sh clickhouse 129 | ``` 130 | 131 | As the result, `resources/modules/clickhouse.metabase-driver.jar` should be created. 132 | 133 | For smoke testing, there is a Metabase with the link to the driver available as a Docker container: 134 | 135 | ```bash 136 | docker compose -f modules/drivers/clickhouse/docker-compose.yml up -d metabase 137 | ``` 138 | 139 | It should pick up the driver jar as a volume. 140 | -------------------------------------------------------------------------------- /.docker/setup/setup.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import logging 3 | import os 4 | import pprint 5 | 6 | import requests 7 | 8 | host = os.environ.get('host') if os.environ.get('host') else 'http://localhost' 9 | port = os.environ.get('port') if os.environ.get('port') else '3000' 10 | admin_email = os.environ.get('admin_email') if os.environ.get('admin_email') else 'admin@example.com' 11 | user_email = os.environ.get('user_email') if os.environ.get('user_email') else 'user@example.com' 12 | password = os.environ.get('password') if os.environ.get('password') else 'metabot1' 13 | site_name = 'ClickHouse test' 14 | 15 | endpoints = { 16 | 'health_check': '/api/health', 17 | 'properties': '/api/session/properties', 18 | 'setup': '/api/setup', 19 | 'database': '/api/database', 20 | 'login': '/api/session', 21 | 'user': '/api/user', 22 | } 23 | for k, v in endpoints.items(): 24 | endpoints[k] = f"{host}:{port}{v}" 25 | 26 | db_base_payload = { 27 | "is_on_demand": False, 28 | "is_full_sync": True, 29 | "is_sample": False, 30 | "cache_ttl": None, 31 | "refingerprint": False, 32 | "auto_run_queries": True, 33 | "schedules": {}, 34 | "details": { 35 | "host": "clickhouse", 36 | "port": 8123, 37 | "user": "default", 38 | "password": None, 39 | "dbname": "default", 40 | "scan-all-databases": False, 41 | "ssl": False, 42 | "tunnel-enabled": False, 43 | "advanced-options": False 44 | }, 45 | "name": "Our ClickHouse", 46 | "engine": "clickhouse" 47 | } 48 | 49 | 50 | def health(): 51 | response = requests.get(endpoints['health_check'], verify=False) 52 | if response.json()['status'] == 'ok': 53 | return 'healthy' 54 | else: 55 | health() 56 | 57 | 58 | def check_response(response, op): 59 | if response.status_code >= 300: 60 | print(f'Unexpected status {response.status_code} for {op}', response.text) 61 | exit(1) 62 | 63 | 64 | if __name__ == '__main__': 65 | print("Checking health") 66 | 67 | if health() == 'healthy' and os.environ.get('retry') is None: 68 | print("Healthy, setting up Metabase") 69 | 70 | session = requests.Session() 71 | session_token = None 72 | try: 73 | token = session.get(endpoints['properties'], verify=False).json()['setup-token'] 74 | setup_payload = { 75 | 'token': f'{token}', 76 | 'user': { 77 | 'first_name': 'Admin', 78 | 'last_name': 'Admin', 79 | 'email': admin_email, 80 | 'site_name': site_name, 81 | 'password': password, 82 | 'password_confirm': password 83 | }, 84 | 'database': None, 85 | 'invite': None, 86 | 'prefs': { 87 | 'site_name': site_name, 88 | 'site_locale': 'en', 89 | 'allow_tracking': False 90 | } 91 | } 92 | print("Getting the setup token") 93 | session_token = session.post(endpoints['setup'], verify=False, json=setup_payload).json()['id'] 94 | except Exception as e: 95 | print("The admin user was already created") 96 | 97 | try: 98 | if session_token is None: 99 | session_token = session.post(endpoints['login'], verify=False, 100 | json={"username": admin_email, "password": password}) 101 | 102 | dbs = session.get(endpoints['database'], verify=False).json() 103 | print("Current databases:") 104 | pprint.pprint(dbs['data']) 105 | 106 | sample_db = next((x for x in dbs['data'] if x['id'] == 1), None) 107 | if sample_db is not None: 108 | print("Deleting the sample database") 109 | res = session.delete(f"{endpoints['database']}/{sample_db['id']}") 110 | check_response(res, 'delete sample db') 111 | else: 112 | print("The sample database was already deleted") 113 | 114 | single_node_db = next((x for x in dbs['data'] 115 | if x['engine'] == 'clickhouse' 116 | and x['details']['host'] == 'clickhouse'), None) 117 | if single_node_db is None: 118 | print("Creating ClickHouse single node db") 119 | single_node_payload = copy.deepcopy(db_base_payload) 120 | single_node_payload['name'] = 'ClickHouse (single node)' 121 | res = session.post(endpoints['database'], verify=False, json=single_node_payload) 122 | check_response(res, 'create single node db') 123 | else: 124 | print("The single node database was already created") 125 | 126 | # cluster_db = next((x for x in dbs['data'] 127 | # if x['engine'] == 'clickhouse' 128 | # and x['details']['host'] == 'nginx'), None) 129 | # if cluster_db is None: 130 | # print("Creating ClickHouse cluster db") 131 | # cluster_db_payload = copy.deepcopy(db_base_payload) 132 | # cluster_db_payload['details']['host'] = 'nginx' 133 | # cluster_db_payload['name'] = 'ClickHouse (cluster)' 134 | # res = session.post(endpoints['database'], verify=False, json=cluster_db_payload) 135 | # check_response(res) 136 | # else: 137 | # print("The cluster database was already created") 138 | 139 | print("Creating a regular user") 140 | user_payload = {"first_name": "User", "last_name": "User", "email": user_email, "password": password} 141 | res = session.post(endpoints['user'], verify=False, json=user_payload) 142 | check_response(res, 'create user') 143 | 144 | print("Done!") 145 | except Exception as e: 146 | logging.exception("Failed to setup Metabase", e) 147 | exit() 148 | -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: Check 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - master 8 | paths-ignore: 9 | - '**/*.md' 10 | pull_request: 11 | paths-ignore: 12 | - '**/*.md' 13 | 14 | env: 15 | # Using a fork to disable a few failing tests 16 | METABASE_REPOSITORY: slvrtrn/metabase 17 | METABASE_VERSION: 0.53.x-ch 18 | 19 | jobs: 20 | check-local-current-version: 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: Checkout Metabase Repo 24 | uses: actions/checkout@v4 25 | with: 26 | repository: ${{ env.METABASE_REPOSITORY }} 27 | ref: ${{ env.METABASE_VERSION }} 28 | 29 | - name: Checkout Driver Repo 30 | uses: actions/checkout@v4 31 | with: 32 | path: modules/drivers/clickhouse 33 | 34 | - name: Prepare JDK 21 35 | uses: actions/setup-java@v4 36 | with: 37 | distribution: "temurin" 38 | java-version: "21" 39 | 40 | - name: Add ClickHouse TLS instance to /etc/hosts 41 | run: | 42 | sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts 43 | 44 | - name: Start ClickHouse in Docker 45 | uses: hoverkraft-tech/compose-action@v2.0.0 46 | with: 47 | compose-file: "modules/drivers/clickhouse/docker-compose.yml" 48 | down-flags: "--volumes" 49 | services: | 50 | clickhouse 51 | clickhouse_tls 52 | clickhouse_older_version 53 | clickhouse_cluster_node1 54 | clickhouse_cluster_node2 55 | nginx 56 | 57 | - name: Install Clojure CLI 58 | run: | 59 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh && 60 | sudo bash ./linux-install-1.11.1.1182.sh 61 | 62 | - name: Setup Node 63 | uses: actions/setup-node@v4 64 | with: 65 | node-version: "20" 66 | cache: "yarn" 67 | 68 | - name: Get M2 cache 69 | uses: actions/cache@v4 70 | with: 71 | path: | 72 | ~/.m2 73 | ~/.gitlibs 74 | key: ${{ runner.os }}-clickhouse-${{ hashFiles('**/deps.edn') }} 75 | 76 | - name: Prepare stuff for pulses 77 | run: yarn build-static-viz 78 | 79 | # Use custom deps.edn containing "user/clickhouse" alias to include driver sources 80 | - name: Prepare deps.edn 81 | run: | 82 | mkdir -p /home/runner/.config/clojure 83 | cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn 84 | 85 | - name: Run all tests with the latest ClickHouse version 86 | env: 87 | DRIVERS: clickhouse 88 | run: | 89 | clojure -X:ci:dev:ee:ee-dev:drivers:drivers-dev:test:user/clickhouse 90 | 91 | check-local-older-version: 92 | runs-on: ubuntu-latest 93 | steps: 94 | - name: Checkout Metabase Repo 95 | uses: actions/checkout@v4 96 | with: 97 | repository: ${{ env.METABASE_REPOSITORY }} 98 | ref: ${{ env.METABASE_VERSION }} 99 | 100 | - name: Checkout Driver Repo 101 | uses: actions/checkout@v4 102 | with: 103 | path: modules/drivers/clickhouse 104 | 105 | - name: Prepare JDK 21 106 | uses: actions/setup-java@v4 107 | with: 108 | distribution: "temurin" 109 | java-version: "21" 110 | 111 | - name: Add ClickHouse TLS instance to /etc/hosts 112 | run: | 113 | sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts 114 | 115 | - name: Start ClickHouse in Docker 116 | uses: hoverkraft-tech/compose-action@v2.0.0 117 | with: 118 | compose-file: "modules/drivers/clickhouse/docker-compose.yml" 119 | down-flags: "--volumes" 120 | services: | 121 | clickhouse 122 | clickhouse_tls 123 | clickhouse_older_version 124 | clickhouse_cluster_node1 125 | clickhouse_cluster_node2 126 | nginx 127 | 128 | - name: Install Clojure CLI 129 | run: | 130 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh && 131 | sudo bash ./linux-install-1.11.1.1182.sh 132 | 133 | - name: Get M2 cache 134 | uses: actions/cache@v4 135 | with: 136 | path: | 137 | ~/.m2 138 | ~/.gitlibs 139 | key: ${{ runner.os }}-clickhouse-${{ hashFiles('**/deps.edn') }} 140 | 141 | # Use custom deps.edn containing "user/clickhouse" alias to include driver sources 142 | - name: Prepare deps.edn 143 | run: | 144 | mkdir -p /home/runner/.config/clojure 145 | cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn 146 | 147 | - name: Run ClickHouse driver tests with 23.3 148 | env: 149 | DRIVERS: clickhouse 150 | MB_CLICKHOUSE_TEST_PORT: 8124 151 | run: | 152 | clojure -X:ci:dev:ee:ee-dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test 153 | 154 | build-jar: 155 | runs-on: ubuntu-latest 156 | needs: [ 'check-local-current-version' ] 157 | steps: 158 | - name: Checkout Metabase Repo 159 | uses: actions/checkout@v4 160 | with: 161 | repository: ${{ env.METABASE_REPOSITORY }} 162 | ref: ${{ env.METABASE_VERSION }} 163 | 164 | - name: Checkout Driver Repo 165 | uses: actions/checkout@v4 166 | with: 167 | path: modules/drivers/clickhouse 168 | 169 | - name: Install Clojure CLI 170 | run: | 171 | curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh && 172 | sudo bash ./linux-install-1.11.1.1182.sh 173 | 174 | - name: Build ClickHouse driver 175 | run: | 176 | echo "{:deps {metabase/clickhouse {:local/root \"clickhouse\" }}}" > modules/drivers/deps.edn 177 | bin/build-driver.sh clickhouse 178 | ls -lah resources/modules 179 | 180 | - name: Archive driver JAR 181 | uses: actions/upload-artifact@v4 182 | with: 183 | name: clickhouse.metabase-driver.jar 184 | path: resources/modules/clickhouse.metabase-driver.jar 185 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | 3 | ########################################################################################################## 4 | # ClickHouse single node (CH driver + Metabase tests) 5 | ########################################################################################################## 6 | 7 | clickhouse: 8 | image: 'clickhouse/clickhouse-server:25.2-alpine' 9 | container_name: 'metabase-driver-clickhouse-server' 10 | hostname: clickhouse 11 | ports: 12 | - '8123:8123' 13 | - '9000:9000' 14 | environment: 15 | CLICKHOUSE_SKIP_USER_SETUP: 1 16 | ulimits: 17 | nofile: 18 | soft: 262144 19 | hard: 262144 20 | volumes: 21 | - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml' 22 | - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml' 23 | 24 | ########################################################################################################## 25 | # ClickHouse single node (CH driver TLS tests only) 26 | ########################################################################################################## 27 | 28 | clickhouse_tls: 29 | build: 30 | context: ./ 31 | dockerfile: .docker/clickhouse/single_node_tls/Dockerfile 32 | container_name: 'metabase-driver-clickhouse-server-tls' 33 | ports: 34 | - '8443:8443' 35 | - '9440:9440' 36 | environment: 37 | CLICKHOUSE_SKIP_USER_SETUP: 1 38 | ulimits: 39 | nofile: 40 | soft: 262144 41 | hard: 262144 42 | volumes: 43 | - './.docker/clickhouse/single_node_tls/config.xml:/etc/clickhouse-server/config.xml' 44 | - './.docker/clickhouse/single_node_tls/users.xml:/etc/clickhouse-server/users.xml' 45 | hostname: server.clickhouseconnect.test 46 | 47 | ########################################################################################################## 48 | # Older ClickHouse version (CH driver tests only) 49 | # For testing pre-23.8 string functions switch between UTF8 and non-UTF8 versions (see clickhouse_qp.clj) 50 | ########################################################################################################## 51 | 52 | clickhouse_older_version: 53 | image: 'clickhouse/clickhouse-server:23.3-alpine' 54 | container_name: 'metabase-driver-clickhouse-server-older-version' 55 | hostname: clickhouse.older 56 | ports: 57 | - '8124:8123' 58 | - '9001:9000' 59 | environment: 60 | CLICKHOUSE_SKIP_USER_SETUP: 1 61 | ulimits: 62 | nofile: 63 | soft: 262144 64 | hard: 262144 65 | volumes: 66 | - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml' 67 | - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml' 68 | 69 | ########################################################################################################## 70 | # ClickHouse cluster (CH driver SET ROLE tests only) 71 | # See test/metabase/driver/clickhouse_set_role.clj 72 | ########################################################################################################## 73 | 74 | clickhouse_cluster_node1: 75 | image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}' 76 | ulimits: 77 | nofile: 78 | soft: 262144 79 | hard: 262144 80 | hostname: clickhouse1 81 | container_name: metabase-driver-clickhouse-cluster-node-1 82 | ports: 83 | - '8125:8123' 84 | - '9002:9000' 85 | - '9181:9181' 86 | environment: 87 | CLICKHOUSE_SKIP_USER_SETUP: 1 88 | volumes: 89 | - './.docker/clickhouse/cluster/server1_config.xml:/etc/clickhouse-server/config.xml' 90 | - './.docker/clickhouse/cluster/server1_macros.xml:/etc/clickhouse-server/config.d/macros.xml' 91 | - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml' 92 | 93 | clickhouse_cluster_node2: 94 | image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}' 95 | ulimits: 96 | nofile: 97 | soft: 262144 98 | hard: 262144 99 | hostname: clickhouse2 100 | container_name: metabase-driver-clickhouse-cluster-node-2 101 | ports: 102 | - '8126:8123' 103 | - '9003:9000' 104 | - '9182:9181' 105 | environment: 106 | CLICKHOUSE_SKIP_USER_SETUP: 1 107 | volumes: 108 | - './.docker/clickhouse/cluster/server2_config.xml:/etc/clickhouse-server/config.xml' 109 | - './.docker/clickhouse/cluster/server2_macros.xml:/etc/clickhouse-server/config.d/macros.xml' 110 | - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml' 111 | 112 | # Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests 113 | # See .docker/nginx/local.conf for the configuration 114 | nginx: 115 | image: 'nginx:1.23.1-alpine' 116 | hostname: nginx 117 | ports: 118 | - '8127:8123' 119 | volumes: 120 | - './.docker/nginx/local.conf:/etc/nginx/conf.d/local.conf' 121 | container_name: metabase-nginx 122 | 123 | ########################################################################################################## 124 | # Metabase 125 | ########################################################################################################## 126 | 127 | metabase: 128 | image: metabase/metabase-enterprise:v1.53.6.4 129 | container_name: metabase-with-clickhouse-driver 130 | hostname: metabase 131 | environment: 132 | 'MB_HTTP_TIMEOUT': '5000' 133 | 'JAVA_TIMEZONE': 'UTC' 134 | ports: 135 | - '3000:3000' 136 | volumes: 137 | - '../../../resources/modules/clickhouse.metabase-driver.jar:/plugins/clickhouse.jar' 138 | - './.docker/clickhouse/single_node_tls/certificates/ca.crt:/certs/ca.crt' 139 | healthcheck: 140 | test: curl --fail -X GET -I http://localhost:3000/api/health || exit 1 141 | interval: 15s 142 | timeout: 5s 143 | retries: 10 144 | 145 | setup: 146 | build: .docker/setup/. 147 | container_name: metabase-clickhouse-setup 148 | volumes: 149 | - .docker/setup/setup.py:/app/setup.py 150 | depends_on: 151 | metabase: 152 | condition: service_healthy 153 | command: python /app/setup.py 154 | environment: 155 | host: http://metabase 156 | port: 3000 157 | admin_email: 'admin@example.com' 158 | user_email: 'user@example.com' 159 | password: 'metabot1' 160 | -------------------------------------------------------------------------------- /test/metabase/driver/clickhouse_temporal_bucketing_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.clickhouse-temporal-bucketing-test 2 | #_{:clj-kondo/ignore [:unsorted-required-namespaces]} 3 | (:require 4 | [clojure.test :refer :all] 5 | [metabase.query-processor.test-util :as qp.test] 6 | [metabase.test :as mt] 7 | [metabase.test.data :as data] 8 | [metabase.test.data.clickhouse :as ctd])) 9 | 10 | (use-fixtures :once ctd/create-test-db!) 11 | 12 | ;; See temporal_bucketing table definition 13 | ;; Fields values are (both in server and column timezones): 14 | ;; start_of_year == '2022-01-01 00:00:00' 15 | ;; mid_of_year == '2022-06-20 06:32:54' 16 | ;; end_of_year == '2022-12-31 23:59:59' 17 | (deftest clickhouse-temporal-bucketing-server-tz 18 | (mt/test-driver 19 | :clickhouse 20 | (defn- start-of-year [unit] 21 | (qp.test/rows 22 | (ctd/do-with-test-db 23 | (fn [db] 24 | (data/with-db db 25 | (data/run-mbql-query 26 | temporal_bucketing_server_tz 27 | {:breakout [[:field %start_of_year {:temporal-unit unit}]]})))))) 28 | (defn- mid-year [unit] 29 | (qp.test/rows 30 | (ctd/do-with-test-db 31 | (fn [db] 32 | (data/with-db db 33 | (data/run-mbql-query 34 | temporal_bucketing_server_tz 35 | {:breakout [[:field %mid_of_year {:temporal-unit unit}]]})))))) 36 | (defn- end-of-year [unit] 37 | (qp.test/rows 38 | (ctd/do-with-test-db 39 | (fn [db] 40 | (data/with-db db 41 | (data/run-mbql-query 42 | temporal_bucketing_server_tz 43 | {:breakout [[:field %end_of_year {:temporal-unit unit}]]})))))) 44 | (testing "truncate to" 45 | (testing "minute" 46 | (is (= [["2022-06-20T06:32:00Z"]] 47 | (mid-year :minute)))) 48 | (testing "hour" 49 | (is (= [["2022-06-20T06:00:00Z"]] 50 | (mid-year :hour)))) 51 | (testing "day" 52 | (is (= [["2022-06-20T00:00:00Z"]] 53 | (mid-year :day)))) 54 | (testing "month" 55 | (is (= [["2022-06-01T00:00:00Z"]] 56 | (mid-year :month)))) 57 | (testing "quarter" 58 | (is (= [["2022-04-01T00:00:00Z"]] 59 | (mid-year :quarter)))) 60 | (testing "year" 61 | (is (= [["2022-01-01T00:00:00Z"]] 62 | (mid-year :year))))) 63 | (testing "extract" 64 | (testing "minute of hour" 65 | (is (= [[0]] 66 | (start-of-year :minute-of-hour))) 67 | (is (= [[32]] 68 | (mid-year :minute-of-hour))) 69 | (is (= [[59]] 70 | (end-of-year :minute-of-hour)))) 71 | (testing "hour of day" 72 | (is (= [[0]] 73 | (start-of-year :hour-of-day))) 74 | (is (= [[6]] 75 | (mid-year :hour-of-day))) 76 | (is (= [[23]] 77 | (end-of-year :hour-of-day)))) 78 | (testing "day of month" 79 | (is (= [[1]] 80 | (start-of-year :day-of-month))) 81 | (is (= [[20]] 82 | (mid-year :day-of-month))) 83 | (is (= [[31]] 84 | (end-of-year :day-of-month)))) 85 | (testing "day of year" 86 | (is (= [[1]] 87 | (start-of-year :day-of-year))) 88 | (is (= [[171]] 89 | (mid-year :day-of-year))) 90 | (is (= [[365]] 91 | (end-of-year :day-of-year)))) 92 | (testing "month of year" 93 | (is (= [[1]] 94 | (start-of-year :month-of-year))) 95 | (is (= [[6]] 96 | (mid-year :month-of-year))) 97 | (is (= [[12]] 98 | (end-of-year :month-of-year)))) 99 | (testing "quarter of year" 100 | (is (= [[1]] 101 | (start-of-year :quarter-of-year))) 102 | (is (= [[2]] 103 | (mid-year :quarter-of-year))) 104 | (is (= [[4]] 105 | (end-of-year :quarter-of-year))))))) 106 | 107 | (deftest clickhouse-temporal-bucketing-column-tz 108 | (mt/test-driver 109 | :clickhouse 110 | (defn- start-of-year [unit] 111 | (qp.test/rows 112 | (ctd/do-with-test-db 113 | (fn [db] 114 | (data/with-db db 115 | (data/run-mbql-query 116 | temporal_bucketing_column_tz 117 | {:breakout [[:field %start_of_year {:temporal-unit unit}]]})))))) 118 | (defn- mid-year [unit] 119 | (qp.test/rows 120 | (ctd/do-with-test-db 121 | (fn [db] 122 | (data/with-db db 123 | (data/run-mbql-query 124 | temporal_bucketing_column_tz 125 | {:breakout [[:field %mid_of_year {:temporal-unit unit}]]})))))) 126 | (defn- end-of-year [unit] 127 | (qp.test/rows 128 | (ctd/do-with-test-db 129 | (fn [db] 130 | (data/with-db db 131 | (data/run-mbql-query 132 | temporal_bucketing_column_tz 133 | {:breakout [[:field %end_of_year {:temporal-unit unit}]]})))))) 134 | (testing "truncate to" 135 | (testing "minute" 136 | (is (= [["2022-06-20T13:32:00Z"]] 137 | (mid-year :minute)))) 138 | (testing "hour" 139 | (is (= [["2022-06-20T13:00:00Z"]] 140 | (mid-year :hour)))) 141 | (testing "day" 142 | (is (= [["2022-06-20T07:00:00Z"]] 143 | (mid-year :day)))) 144 | (testing "month" 145 | (is (= [["2022-06-01T00:00:00Z"]] 146 | (mid-year :month)))) 147 | (testing "quarter" 148 | (is (= [["2022-04-01T00:00:00Z"]] 149 | (mid-year :quarter)))) 150 | (testing "year" 151 | (is (= [["2022-01-01T00:00:00Z"]] 152 | (mid-year :year))))) 153 | (testing "extract" 154 | (testing "minute of hour" 155 | (is (= [[0]] 156 | (start-of-year :minute-of-hour))) 157 | (is (= [[32]] 158 | (mid-year :minute-of-hour))) 159 | (is (= [[59]] 160 | (end-of-year :minute-of-hour)))) 161 | (testing "hour of day" 162 | (is (= [[0]] 163 | (start-of-year :hour-of-day))) 164 | (is (= [[6]] 165 | (mid-year :hour-of-day))) 166 | (is (= [[23]] 167 | (end-of-year :hour-of-day)))) 168 | (testing "day of month" 169 | (is (= [[1]] 170 | (start-of-year :day-of-month))) 171 | (is (= [[20]] 172 | (mid-year :day-of-month))) 173 | (is (= [[31]] 174 | (end-of-year :day-of-month)))) 175 | (testing "day of year" 176 | (is (= [[1]] 177 | (start-of-year :day-of-year))) 178 | (is (= [[171]] 179 | (mid-year :day-of-year))) 180 | (is (= [[365]] 181 | (end-of-year :day-of-year)))) 182 | (testing "month of year" 183 | (is (= [[1]] 184 | (start-of-year :month-of-year))) 185 | (is (= [[6]] 186 | (mid-year :month-of-year))) 187 | (is (= [[12]] 188 | (end-of-year :month-of-year)))) 189 | (testing "quarter of year" 190 | (is (= [[1]] 191 | (start-of-year :quarter-of-year))) 192 | (is (= [[2]] 193 | (mid-year :quarter-of-year))) 194 | (is (= [[4]] 195 | (end-of-year :quarter-of-year))))))) 196 | -------------------------------------------------------------------------------- /src/metabase/driver/clickhouse_introspection.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.clickhouse-introspection 2 | (:require [clojure.java.jdbc :as jdbc] 3 | [clojure.string :as str] 4 | [metabase.config :as config] 5 | [metabase.driver :as driver] 6 | [metabase.driver.ddl.interface :as ddl.i] 7 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] 8 | [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync] 9 | [metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table] 10 | [metabase.util :as u]) 11 | (:import (java.sql DatabaseMetaData))) 12 | 13 | (set! *warn-on-reflection* true) 14 | 15 | (def ^:private database-type->base-type 16 | (sql-jdbc.sync/pattern-based-database-type->base-type 17 | [[#"array" :type/Array] 18 | [#"bool" :type/Boolean] 19 | [#"date" :type/Date] 20 | [#"date32" :type/Date] 21 | [#"decimal" :type/Decimal] 22 | [#"enum8" :type/Text] 23 | [#"enum16" :type/Text] 24 | [#"fixedstring" :type/TextLike] 25 | [#"float32" :type/Float] 26 | [#"float64" :type/Float] 27 | [#"int8" :type/Integer] 28 | [#"int16" :type/Integer] 29 | [#"int32" :type/Integer] 30 | [#"int64" :type/BigInteger] 31 | [#"ipv4" :type/IPAddress] 32 | [#"ipv6" :type/IPAddress] 33 | [#"map" :type/Dictionary] 34 | [#"string" :type/Text] 35 | [#"tuple" :type/*] 36 | [#"uint8" :type/Integer] 37 | [#"uint16" :type/Integer] 38 | [#"uint32" :type/Integer] 39 | [#"uint64" :type/BigInteger] 40 | [#"uuid" :type/UUID]])) 41 | 42 | (defn- normalize-db-type 43 | [db-type] 44 | (cond 45 | ;; LowCardinality 46 | (str/starts-with? db-type "lowcardinality") 47 | (normalize-db-type (subs db-type 15 (- (count db-type) 1))) 48 | ;; Nullable 49 | (str/starts-with? db-type "nullable") 50 | (normalize-db-type (subs db-type 9 (- (count db-type) 1))) 51 | ;; for test purposes only: GMT0 is a legacy timezone; 52 | ;; it maps to LocalDateTime instead of OffsetDateTime 53 | ;; (= db-type "datetime64(3, 'gmt0')") 54 | ;; :type/DateTime 55 | ;; DateTime64 56 | (str/starts-with? db-type "datetime64") 57 | :type/DateTimeWithLocalTZ 58 | ;; DateTime 59 | (str/starts-with? db-type "datetime") 60 | :type/DateTimeWithLocalTZ 61 | ;; Enum* 62 | (str/starts-with? db-type "enum") 63 | :type/Text 64 | ;; Map 65 | (str/starts-with? db-type "map") 66 | :type/Dictionary 67 | ;; Tuple 68 | (str/starts-with? db-type "tuple") 69 | :type/* 70 | ;; SimpleAggregateFunction 71 | (str/starts-with? db-type "simpleaggregatefunction") 72 | (normalize-db-type (subs db-type (+ (str/index-of db-type ",") 2) (- (count db-type) 1))) 73 | ;; _ 74 | :else (or (database-type->base-type (keyword db-type)) :type/*))) 75 | 76 | ;; Enum8(UInt8) -> :type/Text, DateTime64(Europe/Amsterdam) -> :type/DateTime, 77 | ;; Nullable(DateTime) -> :type/DateTime, SimpleAggregateFunction(sum, Int64) -> :type/BigInteger, etc 78 | (defmethod sql-jdbc.sync/database-type->base-type :clickhouse 79 | [_ database-type] 80 | (let [db-type (if (keyword? database-type) 81 | (subs (str database-type) 1) 82 | database-type)] 83 | (normalize-db-type (u/lower-case-en db-type)))) 84 | 85 | (defmethod sql-jdbc.sync/excluded-schemas :clickhouse [_] 86 | #{"system" "information_schema" "INFORMATION_SCHEMA"}) 87 | 88 | (def ^:private allowed-table-types 89 | (into-array String 90 | ["TABLE" "VIEW" "FOREIGN TABLE" "REMOTE TABLE" "DICTIONARY" 91 | "MATERIALIZED VIEW" "MEMORY TABLE" "LOG TABLE"])) 92 | 93 | (defn- tables-set 94 | [tables] 95 | (set 96 | (for [table tables] 97 | (let [remarks (:remarks table)] 98 | {:name (:table_name table) 99 | :schema (:table_schem table) 100 | :description (when-not (str/blank? remarks) remarks)})))) 101 | 102 | (defn- get-tables-from-metadata 103 | [^DatabaseMetaData metadata schema-pattern] 104 | (.getTables metadata 105 | nil ; catalog - unused in the source code there 106 | schema-pattern 107 | "%" ; tablePattern "%" = match all tables 108 | allowed-table-types)) 109 | 110 | (defn- not-inner-mv-table? 111 | [table] 112 | (not (str/starts-with? (:table_name table) ".inner"))) 113 | 114 | (defn- ->spec 115 | [db] 116 | (if (u/id db) 117 | (sql-jdbc.conn/db->pooled-connection-spec db) db)) 118 | 119 | (defn- get-all-tables 120 | [db] 121 | (jdbc/with-db-metadata [metadata (->spec db)] 122 | (->> (get-tables-from-metadata metadata "%") 123 | (jdbc/metadata-result) 124 | (vec) 125 | (filter #(and 126 | (not (contains? (sql-jdbc.sync/excluded-schemas :clickhouse) (:table_schem %))) 127 | (not-inner-mv-table? %))) 128 | (tables-set)))) 129 | 130 | ;; Strangely enough, the tests only work with :db keyword, 131 | ;; but the actual sync from the UI uses :dbname 132 | (defn- get-db-name 133 | [db] 134 | (or (get-in db [:details :dbname]) 135 | (get-in db [:details :db]))) 136 | 137 | (defn- get-tables-in-dbs [db-or-dbs] 138 | (->> (for [db (as-> (or (get-db-name db-or-dbs) "default") dbs 139 | (str/split dbs #" ") 140 | (remove empty? dbs) 141 | (map (comp #(ddl.i/format-name :clickhouse %) str/trim) dbs))] 142 | (jdbc/with-db-metadata [metadata (->spec db-or-dbs)] 143 | (jdbc/metadata-result 144 | (get-tables-from-metadata metadata db)))) 145 | (apply concat) 146 | (filter not-inner-mv-table?) 147 | (tables-set))) 148 | 149 | (defmethod driver/describe-database :clickhouse 150 | [_ {{:keys [scan-all-databases]} 151 | :details :as db}] 152 | {:tables 153 | (if 154 | (boolean scan-all-databases) 155 | (get-all-tables db) 156 | (get-tables-in-dbs db))}) 157 | 158 | (defn- ^:private is-db-required? 159 | [field] 160 | (not (str/starts-with? (get field :database-type) "Nullable"))) 161 | 162 | (defmethod driver/describe-table :clickhouse 163 | [_ database table] 164 | (let [table-metadata (sql-jdbc.sync/describe-table :clickhouse database table) 165 | filtered-fields (for [field (:fields table-metadata) 166 | :let [updated-field (update field :database-required 167 | (fn [_] (is-db-required? field)))] 168 | ;; Skip all AggregateFunction (but keeping SimpleAggregateFunction) columns 169 | ;; JDBC does not support that and it crashes the data browser 170 | :when (not (re-matches #"^AggregateFunction\(.+$" 171 | (get field :database-type)))] 172 | updated-field)] 173 | (merge table-metadata {:fields (set filtered-fields)}))) 174 | 175 | (defmethod sql-jdbc.describe-table/get-table-pks :clickhouse 176 | [_driver ^java.sql.Connection conn db-name-or-nil table] 177 | ;; JDBC v2 sets the PKs now, so that :metadata/key-constraints feature should be enabled; 178 | ;; however, enabling :metadata/key-constraints will also enable left-join tests which are currently failing 179 | (if (not config/is-test?) 180 | (sql-jdbc.describe-table/get-table-pks :sql-jdbc conn db-name-or-nil table) 181 | [])) 182 | -------------------------------------------------------------------------------- /test/metabase/test/data/clickhouse_datasets.sql: -------------------------------------------------------------------------------- 1 | DROP DATABASE IF EXISTS `metabase_test`; 2 | CREATE DATABASE `metabase_test`; 3 | 4 | CREATE TABLE `metabase_test`.`metabase_test_lowercases` 5 | ( 6 | id UInt8, 7 | mystring Nullable(String) 8 | ) ENGINE = Memory; 9 | 10 | INSERT INTO `metabase_test`.`metabase_test_lowercases` 11 | VALUES (1, 'Я_1'), (2, 'R'), (3, 'Я_2'), (4, 'Я'), (5, 'я'), (6, NULL); 12 | 13 | CREATE TABLE `metabase_test`.`enums_test` 14 | ( 15 | enum1 Enum8('foo' = 0, 'bar' = 1, 'foo bar' = 2), 16 | enum2 Enum16('click' = 0, 'house' = 1), 17 | enum3 Enum8('qaz' = 42, 'qux' = 23) 18 | ) ENGINE = Memory; 19 | 20 | INSERT INTO `metabase_test`.`enums_test` (enum1, enum2, enum3) 21 | VALUES ('foo', 'house', 'qaz'), 22 | ('foo bar', 'click', 'qux'), 23 | ('bar', 'house', 'qaz'); 24 | 25 | CREATE TABLE `metabase_test`.`ipaddress_test` 26 | ( 27 | ipvfour Nullable(IPv4), 28 | ipvsix Nullable(IPv6) 29 | ) Engine = Memory; 30 | 31 | INSERT INTO `metabase_test`.`ipaddress_test` (ipvfour, ipvsix) 32 | VALUES (toIPv4('127.0.0.1'), toIPv6('0:0:0:0:0:0:0:1')), 33 | (toIPv4('0.0.0.0'), toIPv6('2001:438:ffff:0:0:0:407d:1bc1')), 34 | (null, null); 35 | 36 | CREATE TABLE `metabase_test`.`boolean_test` 37 | ( 38 | ID Int32, 39 | b1 Bool, 40 | b2 Nullable(Bool) 41 | ) ENGINE = Memory; 42 | 43 | INSERT INTO `metabase_test`.`boolean_test` (ID, b1, b2) 44 | VALUES (1, true, true), 45 | (2, false, true), 46 | (3, true, false); 47 | 48 | CREATE TABLE `metabase_test`.`maps_test` 49 | ( 50 | m Map(String, UInt64) 51 | ) ENGINE = Memory; 52 | 53 | INSERT INTO `metabase_test`.`maps_test` 54 | VALUES ({'key1':1,'key2':10}), 55 | ({'key1':2,'key2':20}), 56 | ({'key1':3,'key2':30}); 57 | 58 | 59 | CREATE TABLE `metabase_test`.`array_of_tuples_test` 60 | ( 61 | t Array(Tuple(String, UInt32)) 62 | ) Engine = Memory; 63 | 64 | INSERT INTO `metabase_test`.`array_of_tuples_test` (t) 65 | VALUES ([('foobar', 1234), ('qaz', 0)]), 66 | ([]); 67 | 68 | -- Used for testing that AggregateFunction columns are excluded, 69 | -- while SimpleAggregateFunction columns are preserved 70 | CREATE TABLE `metabase_test`.`aggregate_functions_filter_test` 71 | ( 72 | idx UInt8, 73 | a AggregateFunction(uniq, String), 74 | lowest_value SimpleAggregateFunction(min, UInt8), 75 | count SimpleAggregateFunction(sum, Int64) 76 | ) ENGINE Memory; 77 | 78 | INSERT INTO `metabase_test`.`aggregate_functions_filter_test` 79 | (idx, lowest_value, count) 80 | VALUES (42, 144, 255255); 81 | 82 | -- Materialized views (testing .inner tables exclusion) 83 | CREATE TABLE `metabase_test`.`wikistat` 84 | ( 85 | `date` Date, 86 | `project` LowCardinality(String), 87 | `hits` UInt32 88 | ) ENGINE = Memory; 89 | 90 | CREATE MATERIALIZED VIEW `metabase_test`.`wikistat_mv` ENGINE =Memory AS 91 | SELECT date, project, sum(hits) AS hits 92 | FROM `metabase_test`.`wikistat` 93 | GROUP BY date, project; 94 | 95 | INSERT INTO `metabase_test`.`wikistat` 96 | VALUES (now(), 'foo', 10), 97 | (now(), 'bar', 10), 98 | (now(), 'bar', 20); 99 | 100 | -- Used in sum-where tests 101 | CREATE TABLE `metabase_test`.`sum_if_test_int` 102 | ( 103 | id Int64, 104 | int_value Int64, 105 | discriminator String 106 | ) ENGINE = Memory; 107 | 108 | INSERT INTO `metabase_test`.`sum_if_test_int` 109 | VALUES (1, 1, 'foo'), 110 | (2, 1, 'foo'), 111 | (3, 3, 'bar'), 112 | (4, 5, 'bar'); 113 | 114 | CREATE TABLE `metabase_test`.`sum_if_test_float` 115 | ( 116 | id Int64, 117 | float_value Float64, 118 | discriminator String 119 | ) ENGINE = Memory; 120 | 121 | INSERT INTO `metabase_test`.`sum_if_test_float` 122 | VALUES (1, 1.1, 'foo'), 123 | (2, 1.44, 'foo'), 124 | (3, 3.5, 'bar'), 125 | (4, 5.77, 'bar'); 126 | 127 | -- Temporal bucketing tests 128 | CREATE TABLE `metabase_test`.`temporal_bucketing_server_tz` 129 | ( 130 | start_of_year DateTime, 131 | mid_of_year DateTime, 132 | end_of_year DateTime 133 | ) ENGINE = Memory; 134 | 135 | INSERT INTO `metabase_test`.`temporal_bucketing_server_tz` 136 | VALUES ('2022-01-01 00:00:00', 137 | '2022-06-20 06:32:54', 138 | '2022-12-31 23:59:59'); 139 | 140 | CREATE TABLE `metabase_test`.`temporal_bucketing_column_tz` 141 | ( 142 | start_of_year DateTime('America/Los_Angeles'), 143 | mid_of_year DateTime('America/Los_Angeles'), 144 | end_of_year DateTime('America/Los_Angeles') 145 | ) ENGINE = Memory; 146 | 147 | INSERT INTO `metabase_test`.`temporal_bucketing_column_tz` 148 | VALUES (toDateTime('2022-01-01 00:00:00', 'America/Los_Angeles'), 149 | toDateTime('2022-06-20 06:32:54', 'America/Los_Angeles'), 150 | toDateTime('2022-12-31 23:59:59', 'America/Los_Angeles')); 151 | 152 | CREATE TABLE `metabase_test`.`datetime_diff_nullable` ( 153 | idx Int32, 154 | dt64 Nullable(DateTime64(3, 'UTC')), 155 | dt Nullable(DateTime('UTC')), 156 | d Nullable(Date) 157 | ) ENGINE Memory; 158 | 159 | INSERT INTO `metabase_test`.`datetime_diff_nullable` 160 | VALUES (42, '2022-01-01 00:00:00.000', '2022-06-20 06:32:54', '2022-07-22'), 161 | (43, '2022-01-01 00:00:00.000', NULL, NULL), 162 | (44, NULL, '2022-06-20 06:32:54', '2022-07-22'), 163 | (45, NULL, NULL, NULL); 164 | 165 | DROP DATABASE IF EXISTS `metabase_db_scan_test`; 166 | CREATE DATABASE `metabase_db_scan_test`; 167 | 168 | CREATE TABLE `metabase_db_scan_test`.`table1` (i Int32) ENGINE = Memory; 169 | CREATE TABLE `metabase_db_scan_test`.`table2` (i Int64) ENGINE = Memory; 170 | 171 | -- Base type matching tests 172 | CREATE TABLE `metabase_test`.`enums_base_types` ( 173 | c1 Nullable(Enum8('America/New_York')), 174 | c2 Enum8('BASE TABLE' = 1, 'VIEW' = 2, 'FOREIGN TABLE' = 3, 'LOCAL TEMPORARY' = 4, 'SYSTEM VIEW' = 5), 175 | c3 Enum8('NO', 'YES'), 176 | c4 Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2), 177 | c5 Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2)), 178 | c6 Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2)) 179 | ) ENGINE Memory; 180 | CREATE TABLE `metabase_test`.`date_base_types` ( 181 | c1 Date, 182 | c2 Date32, 183 | c3 Nullable(Date), 184 | c4 Nullable(Date32) 185 | ) ENGINE Memory; 186 | CREATE TABLE `metabase_test`.`datetime_base_types` ( 187 | c1 Nullable(DateTime('America/New_York')), 188 | c2 DateTime('America/New_York'), 189 | c3 DateTime, 190 | c4 DateTime64(3), 191 | c5 DateTime64(9, 'America/New_York'), 192 | c6 Nullable(DateTime64(6, 'America/New_York')), 193 | c7 Nullable(DateTime64(0)), 194 | c8 Nullable(DateTime) 195 | ) ENGINE Memory; 196 | CREATE TABLE `metabase_test`.`integer_base_types` ( 197 | c1 UInt8, 198 | c2 UInt16, 199 | c3 UInt32, 200 | c4 UInt64, 201 | c5 UInt128, 202 | c6 UInt256, 203 | c7 Int8, 204 | c8 Int16, 205 | c9 Int32, 206 | c10 Int64, 207 | c11 Int128, 208 | c12 Int256, 209 | c13 Nullable(Int32) 210 | ) ENGINE Memory; 211 | CREATE TABLE `metabase_test`.`numeric_base_types` ( 212 | c1 Float32, 213 | c2 Float64, 214 | c3 Decimal(4, 2), 215 | c4 Decimal32(7), 216 | c5 Decimal64(12), 217 | c6 Decimal128(24), 218 | c7 Decimal256(42), 219 | c8 Nullable(Float32), 220 | c9 Nullable(Decimal(4, 2)), 221 | c10 Nullable(Decimal256(42)) 222 | ) ENGINE Memory; 223 | CREATE TABLE `metabase_test`.`string_base_types` ( 224 | c1 String, 225 | c2 LowCardinality(String), 226 | c3 FixedString(32), 227 | c4 Nullable(String), 228 | c5 LowCardinality(FixedString(4)) 229 | ) ENGINE Memory; 230 | CREATE TABLE `metabase_test`.`misc_base_types` ( 231 | c1 Boolean, 232 | c2 UUID, 233 | c3 IPv4, 234 | c4 IPv6, 235 | c5 Map(Int32, String), 236 | c6 Nullable(Boolean), 237 | c7 Nullable(UUID), 238 | c8 Nullable(IPv4), 239 | c9 Nullable(IPv6), 240 | c10 Tuple(String, Int32) 241 | ) ENGINE Memory; 242 | CREATE TABLE `metabase_test`.`array_base_types` ( 243 | c1 Array(String), 244 | c2 Array(Nullable(Int32)), 245 | c3 Array(Array(LowCardinality(FixedString(32)))), 246 | c4 Array(Array(Array(String))) 247 | ) ENGINE Memory; 248 | CREATE TABLE `metabase_test`.`low_cardinality_nullable_base_types` ( 249 | c1 LowCardinality(Nullable(String)), 250 | c2 LowCardinality(Nullable(FixedString(16))) 251 | ) ENGINE Memory; 252 | 253 | -- can-connect tests (odd database names) 254 | DROP DATABASE IF EXISTS `Special@Characters~`; 255 | CREATE DATABASE `Special@Characters~`; 256 | 257 | -- arrays inner types test 258 | CREATE TABLE `metabase_test`.`arrays_inner_types` 259 | ( 260 | `arr_str` Array(String), 261 | `arr_nstr` Array(Nullable(String)), 262 | `arr_dec` Array(Decimal(18, 4)), 263 | `arr_ndec` Array(Nullable(Decimal(18, 4))) 264 | ) 265 | ENGINE Memory; 266 | INSERT INTO `metabase_test`.`arrays_inner_types` VALUES ( 267 | ['a', 'b', 'c'], 268 | [NULL, 'd', 'e'], 269 | [1, 2, 3], 270 | [4, NULL, 5] 271 | ); 272 | 273 | CREATE TABLE `metabase_test`.`unsigned_int_types` 274 | ( 275 | `u8` UInt8, 276 | `u16` UInt16, 277 | `u32` UInt32, 278 | `u64` UInt64 279 | ) ENGINE Memory; 280 | INSERT INTO `metabase_test`.`unsigned_int_types` 281 | VALUES (255, 65535, 4294967295, 18446744073709551615); 282 | 283 | CREATE TABLE `metabase_test`.`fixed_strings` 284 | ( 285 | `f1` FixedString(4), 286 | `f2` LowCardinality(FixedString(4)), 287 | `f3` Nullable(FixedString(4)), 288 | `f4` LowCardinality(Nullable(FixedString(4))) 289 | ) ENGINE Memory; 290 | INSERT INTO `metabase_test`.`fixed_strings` 291 | VALUES ('val1', 'val2', 'val3', 'val4'); 292 | -------------------------------------------------------------------------------- /test/metabase/driver/clickhouse_impersonation_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.clickhouse-impersonation-test 2 | "SET ROLE (connection impersonation feature) tests on with single node or on-premise cluster setups." 3 | #_{:clj-kondo/ignore [:unsorted-required-namespaces]} 4 | (:require [clojure.test :refer :all] 5 | [metabase-enterprise.advanced-permissions.api.util-test :as advanced-perms.api.tu] 6 | [metabase.driver :as driver] 7 | [metabase.driver.sql :as driver.sql] 8 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] 9 | [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] 10 | [metabase.query-processor.store :as qp.store] 11 | [metabase.test :as mt] 12 | [metabase.test.data.clickhouse :as ctd] 13 | [metabase.util :as u] 14 | [toucan2.tools.with-temp :as t2.with-temp])) 15 | 16 | ;; 53+ metabase.sync has moved to metabase.sync.core 17 | (try 18 | (require '[metabase.sync :as sync]) 19 | (catch java.io.FileNotFoundException e 20 | (when (re-find #"metabase/sync\.clj" (.getMessage e)) 21 | (require '[metabase.sync.core :as sync])))) 22 | 23 | (set! *warn-on-reflection* true) 24 | 25 | (defn- set-role-test! 26 | [details-map] 27 | (let [default-role (driver.sql/default-database-role :clickhouse nil) 28 | spec (sql-jdbc.conn/connection-details->spec :clickhouse details-map)] 29 | (testing "default role is NONE" 30 | (is (= default-role "NONE"))) 31 | (testing "does not throw with an existing role" 32 | (sql-jdbc.execute/do-with-connection-with-options 33 | :clickhouse spec nil 34 | (fn [^java.sql.Connection conn] 35 | (driver/set-role! :clickhouse conn "metabase_test_role"))) 36 | (is true)) 37 | (testing "does not throw with a role containing hyphens" 38 | (sql-jdbc.execute/do-with-connection-with-options 39 | :clickhouse spec nil 40 | (fn [^java.sql.Connection conn] 41 | (driver/set-role! :clickhouse conn "metabase-test-role"))) 42 | (is true)) 43 | (testing "does not throw with the default role" 44 | (sql-jdbc.execute/do-with-connection-with-options 45 | :clickhouse spec nil 46 | (fn [^java.sql.Connection conn] 47 | (driver/set-role! :clickhouse conn default-role) 48 | (fn [^java.sql.Connection conn] 49 | (driver/set-role! :clickhouse conn default-role) 50 | (with-open [stmt (.prepareStatement conn "SELECT * FROM `metabase_test_role_db`.`some_table` ORDER BY i ASC;") 51 | rset (.executeQuery stmt)] 52 | (is (.next rset) true) 53 | (is (.getInt rset 1) 42) 54 | (is (.next rset) true) 55 | (is (.getInt rset 1) 144) 56 | (is (.next rset) false))))) 57 | (is true)))) 58 | 59 | (defn- set-role-throws-test! 60 | [details-map] 61 | (testing "throws when assigning a non-existent role" 62 | (is (thrown? Exception 63 | (sql-jdbc.execute/do-with-connection-with-options 64 | :clickhouse (sql-jdbc.conn/connection-details->spec :clickhouse details-map) nil 65 | (fn [^java.sql.Connection conn] 66 | (driver/set-role! :clickhouse conn "asdf"))))))) 67 | 68 | (defn- do-with-new-metadata-provider 69 | [details thunk] 70 | (t2.with-temp/with-temp 71 | [:model/Database db {:engine :clickhouse :details details}] 72 | (qp.store/with-metadata-provider (u/the-id db) (thunk db)))) 73 | 74 | (deftest clickhouse-set-role 75 | (mt/test-driver 76 | :clickhouse 77 | (let [user-details {:user "metabase_test_user"} 78 | ;; See docker-compose.yml for the port mappings 79 | ;; 24.4+ 80 | single-node-port-details {:port 8123} 81 | single-node-details (merge user-details single-node-port-details) 82 | cluster-port-details {:port 8127} 83 | cluster-details (merge user-details cluster-port-details)] 84 | (testing "single node" 85 | (testing "should support the impersonation feature" 86 | (t2.with-temp/with-temp 87 | [:model/Database db {:engine :clickhouse :details {:user "default" :port 8123}}] 88 | (is (true? (driver/database-supports? :clickhouse :connection-impersonation db))))) 89 | (let [statements ["CREATE DATABASE IF NOT EXISTS `metabase_test_role_db`;" 90 | "CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` (i Int32) ENGINE = MergeTree ORDER BY (i);" 91 | "INSERT INTO `metabase_test_role_db`.`some_table` VALUES (42), (144);" 92 | "CREATE ROLE IF NOT EXISTS `metabase_test_role`;" 93 | "CREATE ROLE IF NOT EXISTS `metabase-test-role`;" 94 | "CREATE USER IF NOT EXISTS `metabase_test_user` NOT IDENTIFIED;" 95 | "GRANT SELECT ON `metabase_test_role_db`.* TO `metabase_test_role`,`metabase-test-role`;" 96 | "GRANT `metabase_test_role`, `metabase-test-role` TO `metabase_test_user`;"]] 97 | (ctd/exec-statements statements single-node-port-details) 98 | (do-with-new-metadata-provider 99 | single-node-details 100 | (fn [_db] 101 | (set-role-test! single-node-details) 102 | (set-role-throws-test! single-node-details))))) 103 | (testing "on-premise cluster" 104 | (testing "should support the impersonation feature" 105 | (t2.with-temp/with-temp 106 | [:model/Database db {:engine :clickhouse :details {:user "default" :port 8127}}] 107 | (is (true? (driver/database-supports? :clickhouse :connection-impersonation db))))) 108 | (let [statements ["CREATE DATABASE IF NOT EXISTS `metabase_test_role_db` ON CLUSTER '{cluster}';" 109 | "CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` ON CLUSTER '{cluster}' (i Int32) 110 | ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}') 111 | ORDER BY (i);" 112 | "INSERT INTO `metabase_test_role_db`.`some_table` VALUES (42), (144);" 113 | "CREATE ROLE IF NOT EXISTS `metabase_test_role` ON CLUSTER '{cluster}';" 114 | "CREATE ROLE IF NOT EXISTS `metabase-test-role` ON CLUSTER '{cluster}';" 115 | "CREATE USER IF NOT EXISTS `metabase_test_user` ON CLUSTER '{cluster}' NOT IDENTIFIED;" 116 | "GRANT ON CLUSTER '{cluster}' SELECT ON `metabase_test_role_db`.* TO `metabase_test_role`, `metabase-test-role`;" 117 | "GRANT ON CLUSTER '{cluster}' `metabase_test_role`, `metabase-test-role` TO `metabase_test_user`;"]] 118 | (ctd/exec-statements statements cluster-port-details) 119 | (do-with-new-metadata-provider 120 | cluster-details 121 | (fn [_db] 122 | (set-role-test! cluster-details) 123 | (set-role-throws-test! cluster-details))))) 124 | (testing "older ClickHouse version" ;; 23.3 125 | (testing "should NOT support the impersonation feature" 126 | (t2.with-temp/with-temp 127 | [:model/Database db {:engine :clickhouse :details {:user "default" :port 8124}}] 128 | (is (false? (driver/database-supports? :clickhouse :connection-impersonation db))))))))) 129 | 130 | (deftest conn-impersonation-test-clickhouse 131 | (mt/test-driver 132 | :clickhouse 133 | (mt/with-premium-features #{:advanced-permissions} 134 | (let [table-name (str "metabase_impersonation_test.test_" (System/currentTimeMillis)) 135 | select-query (format "SELECT * FROM %s;" table-name) 136 | cluster-port {:port 8127} 137 | cluster-details {:engine :clickhouse 138 | :details {:user "metabase_impersonation_test_user" 139 | :dbname "metabase_impersonation_test" 140 | :port 8127}} 141 | ddl-statements ["CREATE DATABASE IF NOT EXISTS metabase_impersonation_test ON CLUSTER '{cluster}';" 142 | (format "CREATE TABLE %s ON CLUSTER '{cluster}' (s String) 143 | ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}') 144 | ORDER BY (s);" table-name)] 145 | insert-statements [(format "INSERT INTO %s VALUES ('a'), ('b'), ('c');" table-name)] 146 | grant-statements ["CREATE USER IF NOT EXISTS metabase_impersonation_test_user ON CLUSTER '{cluster}' NOT IDENTIFIED;" 147 | "CREATE ROLE IF NOT EXISTS row_a ON CLUSTER '{cluster}';" 148 | "CREATE ROLE IF NOT EXISTS row_b ON CLUSTER '{cluster}';" 149 | "CREATE ROLE IF NOT EXISTS row_c ON CLUSTER '{cluster}';" 150 | "GRANT ON CLUSTER '{cluster}' row_a, row_b, row_c TO metabase_impersonation_test_user;" 151 | (format "GRANT ON CLUSTER '{cluster}' SELECT ON %s TO metabase_impersonation_test_user;" table-name) 152 | (format "CREATE ROW POLICY OR REPLACE policy_row_a ON CLUSTER '{cluster}' 153 | ON %s FOR SELECT USING s = 'a' TO row_a;" table-name) 154 | (format "CREATE ROW POLICY OR REPLACE policy_row_b ON CLUSTER '{cluster}' 155 | ON %s FOR SELECT USING s = 'b' TO row_b;" table-name) 156 | (format "CREATE ROW POLICY OR REPLACE policy_row_c ON CLUSTER '{cluster}' 157 | ON %s FOR SELECT USING s = 'c' TO row_c;" table-name)]] 158 | (ctd/exec-statements ddl-statements cluster-port {"wait_end_of_query" "1"}) 159 | (ctd/exec-statements insert-statements cluster-port {"wait_end_of_query" "1" 160 | "insert_quorum" "2"}) 161 | (ctd/exec-statements grant-statements cluster-port {"wait_end_of_query" "1"}) 162 | (t2.with-temp/with-temp [:model/Database db cluster-details] 163 | (mt/with-db db (sync/sync-database! db) 164 | 165 | (letfn [(check-impersonation! [roles expected] 166 | (advanced-perms.api.tu/with-impersonations! 167 | {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}] 168 | :attributes {"impersonation_attr" roles}} 169 | (is (= expected 170 | (-> {:query select-query} 171 | mt/native-query 172 | mt/process-query 173 | mt/rows)))))] 174 | 175 | (is (= [["a"] ["b"] ["c"]] 176 | (-> {:query select-query} 177 | mt/native-query 178 | mt/process-query 179 | mt/rows))) 180 | 181 | (check-impersonation! "row_a" [["a"]]) 182 | (check-impersonation! "row_b" [["b"]]) 183 | (check-impersonation! "row_c" [["c"]]) 184 | (check-impersonation! "row_a,row_c" [["a"] ["c"]]) 185 | (check-impersonation! "row_b,row_c" [["b"] ["c"]]) 186 | (check-impersonation! "row_a,row_b,row_c" [["a"] ["b"] ["c"]])))))))) 187 | -------------------------------------------------------------------------------- /test/metabase/driver/clickhouse_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.clickhouse-test 2 | "Tests for specific behavior of the ClickHouse driver." 3 | #_{:clj-kondo/ignore [:unsorted-required-namespaces]} 4 | (:require [clojure.test :refer :all] 5 | [metabase.driver :as driver] 6 | [metabase.driver.clickhouse :as clickhouse] 7 | [metabase.driver.clickhouse-qp :as clickhouse-qp] 8 | [metabase.driver.sql-jdbc :as sql-jdbc] 9 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] 10 | [metabase.query-processor.compile :as qp.compile] 11 | [metabase.test :as mt] 12 | [metabase.test.data :as data] 13 | [metabase.test.data.interface :as tx] 14 | [metabase.test.data.clickhouse :as ctd] 15 | [taoensso.nippy :as nippy] 16 | [toucan2.tools.with-temp :as t2.with-temp])) 17 | 18 | (set! *warn-on-reflection* true) 19 | 20 | (use-fixtures :once ctd/create-test-db!) 21 | 22 | ;; the mt/with-dynamic-redefs macro was renamed to mt/with-dynamic-fn-redefs for 0.53+ 23 | ;; as 0.52 is still tested by CI we will check which macro is defined and use that 24 | (defmacro with-dynamic-redefs [bindings & body] 25 | (if (resolve `mt/with-dynamic-redefs) 26 | `(mt/with-dynamic-redefs ~bindings ~@body) 27 | `(mt/with-dynamic-fn-redefs ~bindings ~@body))) 28 | 29 | (deftest ^:parallel clickhouse-version 30 | (mt/test-driver 31 | :clickhouse 32 | (t2.with-temp/with-temp 33 | [:model/Database db 34 | {:engine :clickhouse 35 | :details (tx/dbdef->connection-details :clickhouse :db {:database-name "default"})}] 36 | (let [version (driver/dbms-version :clickhouse db)] 37 | (is (number? (get-in version [:semantic-version :major]))) 38 | (is (number? (get-in version [:semantic-version :minor]))) 39 | (is (string? (get version :version))))))) 40 | 41 | (deftest ^:parallel clickhouse-server-timezone 42 | (mt/test-driver 43 | :clickhouse 44 | (is (= "UTC" 45 | (let [details (tx/dbdef->connection-details :clickhouse :db {:database-name "default"}) 46 | spec (sql-jdbc.conn/connection-details->spec :clickhouse details)] 47 | (driver/db-default-timezone :clickhouse spec)))))) 48 | 49 | (deftest ^:parallel clickhouse-connection-string 50 | (mt/with-dynamic-fn-redefs [ ;; This function's implementation requires the connection details to actually connect to the 51 | ;; database, which is orthogonal to the purpose of this test. 52 | clickhouse/cloud? (constantly false)] 53 | (testing "connection with no additional options" 54 | (is (= ctd/default-connection-params 55 | (sql-jdbc.conn/connection-details->spec 56 | :clickhouse 57 | {})))) 58 | (testing "custom connection with additional options" 59 | (is (= (merge 60 | ctd/default-connection-params 61 | {:subname "//myclickhouse:9999/foo?sessionTimeout=42" 62 | :user "bob" 63 | :password "qaz" 64 | :ssl true 65 | :custom_http_params "max_threads=42,allow_experimental_analyzer=0"}) 66 | (sql-jdbc.conn/connection-details->spec 67 | :clickhouse 68 | {:host "myclickhouse" 69 | :port 9999 70 | :user "bob" 71 | :password "qaz" 72 | :dbname "foo" 73 | :additional-options "sessionTimeout=42" 74 | :ssl true 75 | :clickhouse-settings "max_threads=42,allow_experimental_analyzer=0"})))) 76 | (testing "nil dbname handling" 77 | (is (= ctd/default-connection-params 78 | (sql-jdbc.conn/connection-details->spec 79 | :clickhouse {:dbname nil})))) 80 | (testing "schema removal" 81 | (doall 82 | (for [host ["localhost" "http://localhost" "https://localhost"]] 83 | (testing (str "for host " host) 84 | (is (= ctd/default-connection-params 85 | (sql-jdbc.conn/connection-details->spec 86 | :clickhouse {:host host})))))) 87 | (doall 88 | (for [host ["myhost" "http://myhost" "https://myhost"]] 89 | (testing (str "for host " host) 90 | (is (= (merge ctd/default-connection-params 91 | {:subname "//myhost:8123/default"}) 92 | (sql-jdbc.conn/connection-details->spec 93 | :clickhouse {:host host})))))) 94 | (doall 95 | (for [host ["sub.example.com" "http://sub.example.com" "https://sub.example.com"]] 96 | (testing (str "for host " host " with some additional params") 97 | (is (= (merge ctd/default-connection-params 98 | {:subname "//sub.example.com:8443/mydb" :ssl true}) 99 | (sql-jdbc.conn/connection-details->spec 100 | :clickhouse {:host host :dbname "mydb" :port 8443 :ssl true}))))))))) 101 | 102 | (deftest ^:parallel clickhouse-connection-string-select-sequential-consistency 103 | (mt/with-dynamic-fn-redefs [ ;; This function's implementation requires the connection details to actually 104 | ;; connect to the database, which is orthogonal to the purpose of this test. 105 | clickhouse/cloud? (constantly true)] 106 | (testing "connection with no additional options" 107 | (is (= (assoc ctd/default-connection-params :select_sequential_consistency true) 108 | (sql-jdbc.conn/connection-details->spec 109 | :clickhouse 110 | {})))))) 111 | 112 | (deftest clickhouse-connection-fails-test 113 | (mt/test-driver 114 | :clickhouse 115 | (mt/with-temp [:model/Database db {:details (assoc (mt/db) :password "wrongpassword") :engine :clickhouse}] 116 | (testing "sense check that checking the cloud mode fails with a SQLException." 117 | ;; nil arg isn't tested here, as it will pick up the defaults, which is the same as the Docker instance credentials. 118 | (is (thrown? java.sql.SQLException (#'clickhouse/cloud? (:details db))))) 119 | (testing "`driver/database-supports? :uploads` does not throw even if the connection fails." 120 | (is (false? (driver/database-supports? :clickhouse :uploads db))) 121 | (is (false? (driver/database-supports? :clickhouse :uploads nil)))) 122 | (testing "`driver/database-supports? :connection-impersonation` does not throw even if the connection fails." 123 | (is (false? (driver/database-supports? :clickhouse :connection-impersonation db))) 124 | (is (false? (driver/database-supports? :clickhouse :connection-impersonation nil)))) 125 | (testing (str "`sql-jdbc.conn/connection-details->spec` does not throw even if the connection fails, " 126 | "and doesn't include the `select_sequential_consistency` parameter.") 127 | (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse db)))) 128 | (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse nil)))))))) 129 | 130 | (deftest ^:parallel clickhouse-tls 131 | (mt/test-driver 132 | :clickhouse 133 | (let [working-dir (System/getProperty "user.dir") 134 | cert-path (str working-dir "/modules/drivers/clickhouse/.docker/clickhouse/single_node_tls/certificates/ca.crt") 135 | additional-options (str "sslrootcert=" cert-path)] 136 | (testing "simple connection with a single database" 137 | (is (= "UTC" 138 | (driver/db-default-timezone 139 | :clickhouse 140 | (sql-jdbc.conn/connection-details->spec 141 | :clickhouse 142 | {:ssl true 143 | :host "server.clickhouseconnect.test" 144 | :port 8443 145 | :additional-options additional-options}))))) 146 | (testing "connection with multiple databases" 147 | (is (= "UTC" 148 | (driver/db-default-timezone 149 | :clickhouse 150 | (sql-jdbc.conn/connection-details->spec 151 | :clickhouse 152 | {:ssl true 153 | :host "server.clickhouseconnect.test" 154 | :port 8443 155 | :dbname "default system" 156 | :additional-options additional-options})))))))) 157 | 158 | (deftest ^:parallel clickhouse-nippy 159 | (mt/test-driver 160 | :clickhouse 161 | (testing "UnsignedByte" 162 | (let [value (com.clickhouse.data.value.UnsignedByte/valueOf "214")] 163 | (is (= value (nippy/thaw (nippy/freeze value)))))) 164 | (testing "UnsignedShort" 165 | (let [value (com.clickhouse.data.value.UnsignedShort/valueOf "62055")] 166 | (is (= value (nippy/thaw (nippy/freeze value)))))) 167 | (testing "UnsignedInteger" 168 | (let [value (com.clickhouse.data.value.UnsignedInteger/valueOf "4748364")] 169 | (is (= value (nippy/thaw (nippy/freeze value)))))) 170 | (testing "UnsignedLong" 171 | (let [value (com.clickhouse.data.value.UnsignedLong/valueOf "84467440737095")] 172 | (is (= value (nippy/thaw (nippy/freeze value)))))))) 173 | 174 | (deftest ^:parallel clickhouse-query-formatting 175 | (mt/test-driver 176 | :clickhouse 177 | (let [query (data/mbql-query venues {:fields [$id] :order-by [[:asc $id]] :limit 5}) 178 | {compiled :query} (qp.compile/compile-with-inline-parameters query) 179 | pretty (driver/prettify-native-form :clickhouse compiled)] 180 | (testing "compiled" 181 | (is (= "SELECT `test_data`.`venues`.`id` AS `id` FROM `test_data`.`venues` ORDER BY `test_data`.`venues`.`id` ASC LIMIT 5" compiled))) 182 | (testing "pretty" 183 | (is (= "SELECT\n `test_data`.`venues`.`id` AS `id`\nFROM\n `test_data`.`venues`\nORDER BY\n `test_data`.`venues`.`id` ASC\nLIMIT\n 5" pretty)))))) 184 | 185 | (deftest ^:parallel clickhouse-can-connect 186 | (mt/test-driver 187 | :clickhouse 188 | (doall 189 | (for [[username password] [["default" ""] ["user_with_password" "foo@bar!"]] 190 | database ["default" "Special@Characters~"]] 191 | (testing (format "User `%s` can connect to `%s`" username database) 192 | (let [details (merge {:user username :password password} 193 | (tx/dbdef->connection-details :clickhouse :db {:database-name database}))] 194 | (is (true? (driver/can-connect? :clickhouse details))))))))) 195 | 196 | (deftest clickhouse-qp-extract-datetime-timezone 197 | (mt/test-driver 198 | :clickhouse 199 | (is (= "utc" (#'clickhouse-qp/extract-datetime-timezone "datetime('utc')"))) 200 | (is (= "utc" (#'clickhouse-qp/extract-datetime-timezone "datetime64(3, 'utc')"))) 201 | (is (= "europe/amsterdam" (#'clickhouse-qp/extract-datetime-timezone "datetime('europe/amsterdam')"))) 202 | (is (= "europe/amsterdam" (#'clickhouse-qp/extract-datetime-timezone "datetime64(9, 'europe/amsterdam')"))) 203 | (is (= nil (#'clickhouse-qp/extract-datetime-timezone "datetime"))) 204 | (is (= nil (#'clickhouse-qp/extract-datetime-timezone "datetime64"))) 205 | (is (= nil (#'clickhouse-qp/extract-datetime-timezone "datetime64(3)"))))) 206 | 207 | (deftest ^:synchronized clickhouse-insert 208 | (mt/test-driver 209 | :clickhouse 210 | (t2.with-temp/with-temp 211 | [:model/Database db 212 | {:engine :clickhouse 213 | :details (tx/dbdef->connection-details :clickhouse :db {:database-name "default"})}] 214 | (let [table (keyword (format "insert_table_%s" (System/currentTimeMillis)))] 215 | (driver/create-table! :clickhouse (:id db) table {:id "Int64", :name "String"}) 216 | (try 217 | (driver/insert-into! :clickhouse (:id db) table [:id :name] [[42 "Bob"] [43 "Alice"]]) 218 | (is (= #{{:id 42, :name "Bob"} 219 | {:id 43, :name "Alice"}} 220 | (set (sql-jdbc/query :clickhouse db {:select [:*] :from [table]})))) 221 | (finally 222 | (driver/drop-table! :clickhouse (:id db) table))))))) 223 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016-2025 ClickHouse, Inc. 2 | 3 | Apache License 4 | Version 2.0, January 2004 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, 12 | and distribution as defined by Sections 1 through 9 of this document. 13 | 14 | "Licensor" shall mean the copyright owner or entity authorized by 15 | the copyright owner that is granting the License. 16 | 17 | "Legal Entity" shall mean the union of the acting entity and all 18 | other entities that control, are controlled by, or are under common 19 | control with that entity. For the purposes of this definition, 20 | "control" means (i) the power, direct or indirect, to cause the 21 | direction or management of such entity, whether by contract or 22 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 23 | outstanding shares, or (iii) beneficial ownership of such entity. 24 | 25 | "You" (or "Your") shall mean an individual or Legal Entity 26 | exercising permissions granted by this License. 27 | 28 | "Source" form shall mean the preferred form for making modifications, 29 | including but not limited to software source code, documentation 30 | source, and configuration files. 31 | 32 | "Object" form shall mean any form resulting from mechanical 33 | transformation or translation of a Source form, including but 34 | not limited to compiled object code, generated documentation, 35 | and conversions to other media types. 36 | 37 | "Work" shall mean the work of authorship, whether in Source or 38 | Object form, made available under the License, as indicated by a 39 | copyright notice that is included in or attached to the work 40 | (an example is provided in the Appendix below). 41 | 42 | "Derivative Works" shall mean any work, whether in Source or Object 43 | form, that is based on (or derived from) the Work and for which the 44 | editorial revisions, annotations, elaborations, or other modifications 45 | represent, as a whole, an original work of authorship. For the purposes 46 | of this License, Derivative Works shall not include works that remain 47 | separable from, or merely link (or bind by name) to the interfaces of, 48 | the Work and Derivative Works thereof. 49 | 50 | "Contribution" shall mean any work of authorship, including 51 | the original version of the Work and any modifications or additions 52 | to that Work or Derivative Works thereof, that is intentionally 53 | submitted to Licensor for inclusion in the Work by the copyright owner 54 | or by an individual or Legal Entity authorized to submit on behalf of 55 | the copyright owner. For the purposes of this definition, "submitted" 56 | means any form of electronic, verbal, or written communication sent 57 | to the Licensor or its representatives, including but not limited to 58 | communication on electronic mailing lists, source code control systems, 59 | and issue tracking systems that are managed by, or on behalf of, the 60 | Licensor for the purpose of discussing and improving the Work, but 61 | excluding communication that is conspicuously marked or otherwise 62 | designated in writing by the copyright owner as "Not a Contribution." 63 | 64 | "Contributor" shall mean Licensor and any individual or Legal Entity 65 | on behalf of whom a Contribution has been received by Licensor and 66 | subsequently incorporated within the Work. 67 | 68 | 2. Grant of Copyright License. Subject to the terms and conditions of 69 | this License, each Contributor hereby grants to You a perpetual, 70 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 71 | copyright license to reproduce, prepare Derivative Works of, 72 | publicly display, publicly perform, sublicense, and distribute the 73 | Work and such Derivative Works in Source or Object form. 74 | 75 | 3. Grant of Patent License. Subject to the terms and conditions of 76 | this License, each Contributor hereby grants to You a perpetual, 77 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 78 | (except as stated in this section) patent license to make, have made, 79 | use, offer to sell, sell, import, and otherwise transfer the Work, 80 | where such license applies only to those patent claims licensable 81 | by such Contributor that are necessarily infringed by their 82 | Contribution(s) alone or by combination of their Contribution(s) 83 | with the Work to which such Contribution(s) was submitted. If You 84 | institute patent litigation against any entity (including a 85 | cross-claim or counterclaim in a lawsuit) alleging that the Work 86 | or a Contribution incorporated within the Work constitutes direct 87 | or contributory patent infringement, then any patent licenses 88 | granted to You under this License for that Work shall terminate 89 | as of the date such litigation is filed. 90 | 91 | 4. Redistribution. You may reproduce and distribute copies of the 92 | Work or Derivative Works thereof in any medium, with or without 93 | modifications, and in Source or Object form, provided that You 94 | meet the following conditions: 95 | 96 | (a) You must give any other recipients of the Work or 97 | Derivative Works a copy of this License; and 98 | 99 | (b) You must cause any modified files to carry prominent notices 100 | stating that You changed the files; and 101 | 102 | (c) You must retain, in the Source form of any Derivative Works 103 | that You distribute, all copyright, patent, trademark, and 104 | attribution notices from the Source form of the Work, 105 | excluding those notices that do not pertain to any part of 106 | the Derivative Works; and 107 | 108 | (d) If the Work includes a "NOTICE" text file as part of its 109 | distribution, then any Derivative Works that You distribute must 110 | include a readable copy of the attribution notices contained 111 | within such NOTICE file, excluding those notices that do not 112 | pertain to any part of the Derivative Works, in at least one 113 | of the following places: within a NOTICE text file distributed 114 | as part of the Derivative Works; within the Source form or 115 | documentation, if provided along with the Derivative Works; or, 116 | within a display generated by the Derivative Works, if and 117 | wherever such third-party notices normally appear. The contents 118 | of the NOTICE file are for informational purposes only and 119 | do not modify the License. You may add Your own attribution 120 | notices within Derivative Works that You distribute, alongside 121 | or as an addendum to the NOTICE text from the Work, provided 122 | that such additional attribution notices cannot be construed 123 | as modifying the License. 124 | 125 | You may add Your own copyright statement to Your modifications and 126 | may provide additional or different license terms and conditions 127 | for use, reproduction, or distribution of Your modifications, or 128 | for any such Derivative Works as a whole, provided Your use, 129 | reproduction, and distribution of the Work otherwise complies with 130 | the conditions stated in this License. 131 | 132 | 5. Submission of Contributions. Unless You explicitly state otherwise, 133 | any Contribution intentionally submitted for inclusion in the Work 134 | by You to the Licensor shall be under the terms and conditions of 135 | this License, without any additional terms or conditions. 136 | Notwithstanding the above, nothing herein shall supersede or modify 137 | the terms of any separate license agreement you may have executed 138 | with Licensor regarding such Contributions. 139 | 140 | 6. Trademarks. This License does not grant permission to use the trade 141 | names, trademarks, service marks, or product names of the Licensor, 142 | except as required for reasonable and customary use in describing the 143 | origin of the Work and reproducing the content of the NOTICE file. 144 | 145 | 7. Disclaimer of Warranty. Unless required by applicable law or 146 | agreed to in writing, Licensor provides the Work (and each 147 | Contributor provides its Contributions) on an "AS IS" BASIS, 148 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 149 | implied, including, without limitation, any warranties or conditions 150 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 151 | PARTICULAR PURPOSE. You are solely responsible for determining the 152 | appropriateness of using or redistributing the Work and assume any 153 | risks associated with Your exercise of permissions under this License. 154 | 155 | 8. Limitation of Liability. In no event and under no legal theory, 156 | whether in tort (including negligence), contract, or otherwise, 157 | unless required by applicable law (such as deliberate and grossly 158 | negligent acts) or agreed to in writing, shall any Contributor be 159 | liable to You for damages, including any direct, indirect, special, 160 | incidental, or consequential damages of any character arising as a 161 | result of this License or out of the use or inability to use the 162 | Work (including but not limited to damages for loss of goodwill, 163 | work stoppage, computer failure or malfunction, or any and all 164 | other commercial damages or losses), even if such Contributor 165 | has been advised of the possibility of such damages. 166 | 167 | 9. Accepting Warranty or Additional Liability. While redistributing 168 | the Work or Derivative Works thereof, You may choose to offer, 169 | and charge a fee for, acceptance of support, warranty, indemnity, 170 | or other liability obligations and/or rights consistent with this 171 | License. However, in accepting such obligations, You may act only 172 | on Your own behalf and on Your sole responsibility, not on behalf 173 | of any other Contributor, and only if You agree to indemnify, 174 | defend, and hold each Contributor harmless for any liability 175 | incurred by, or claims asserted against, such Contributor by reason 176 | of your accepting any such warranty or additional liability. 177 | 178 | END OF TERMS AND CONDITIONS 179 | 180 | APPENDIX: How to apply the Apache License to your work. 181 | 182 | To apply the Apache License to your work, attach the following 183 | boilerplate notice, with the fields enclosed by brackets "[]" 184 | replaced with your own identifying information. (Don't include 185 | the brackets!) The text should be enclosed in the appropriate 186 | comment syntax for the file format. We also recommend that a 187 | file or class name and description of purpose be included on the 188 | same "printed page" as the copyright notice for easier 189 | identification within third-party archives. 190 | 191 | Copyright 2016-2025 ClickHouse, Inc. 192 | 193 | Licensed under the Apache License, Version 2.0 (the "License"); 194 | you may not use this file except in compliance with the License. 195 | You may obtain a copy of the License at 196 | 197 | http://www.apache.org/licenses/LICENSE-2.0 198 | 199 | Unless required by applicable law or agreed to in writing, software 200 | distributed under the License is distributed on an "AS IS" BASIS, 201 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 202 | See the License for the specific language governing permissions and 203 | limitations under the License. 204 | -------------------------------------------------------------------------------- /test/metabase/test/data/clickhouse.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.test.data.clickhouse 2 | "Code for creating / destroying a ClickHouse database from a `DatabaseDefinition`." 3 | (:require 4 | [clojure.java.io :as io] 5 | [clojure.java.jdbc :as jdbc] 6 | [clojure.string :as str] 7 | [clojure.test :refer :all] 8 | [java-time.api :as t] 9 | [metabase.db.query :as mdb.query] 10 | [metabase.driver :as driver] 11 | [metabase.driver.ddl.interface :as ddl.i] 12 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] 13 | [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] 14 | [metabase.driver.sql.util :as sql.u] 15 | [metabase.lib.schema.common :as lib.schema.common] 16 | [metabase.query-processor-test.alternative-date-test :as qp.alternative-date-test] 17 | [metabase.query-processor.test-util :as qp.test] 18 | [metabase.sync.core :as sync] 19 | [metabase.test.data.interface :as tx] 20 | [metabase.test.data.sql :as sql.tx] 21 | [metabase.test.data.sql-jdbc :as sql-jdbc.tx] 22 | [metabase.test.data.sql-jdbc.execute :as execute] 23 | [metabase.test.data.sql-jdbc.load-data :as load-data] 24 | [metabase.test.data.sql.ddl :as ddl] 25 | [metabase.util.log :as log] 26 | [metabase.util.malli :as mu] 27 | [toucan2.tools.with-temp :as t2.with-temp])) 28 | 29 | (sql-jdbc.tx/add-test-extensions! :clickhouse) 30 | 31 | (defmethod driver/database-supports? [:clickhouse :metabase.driver.sql-jdbc.sync.describe-table-test/describe-view-fields] 32 | [_driver _feature _db] true) 33 | (defmethod driver/database-supports? [:clickhouse :metabase.driver.sql-jdbc.sync.describe-table-test/describe-materialized-view-fields] 34 | [_driver _feature _db] false) 35 | 36 | (defmethod driver/database-supports? [:clickhouse :metabase.query-processor-test.parameters-test/get-parameter-count] 37 | [_driver _feature _db] false) 38 | 39 | (defmethod qp.alternative-date-test/iso-8601-text-fields-expected-rows :clickhouse 40 | [_driver] 41 | [[1 "foo" (t/offset-date-time "2004-10-19T10:23:54Z") #t "2004-10-19" (t/offset-date-time "1970-01-01T10:23:54Z")] 42 | [2 "bar" (t/offset-date-time "2008-10-19T10:23:54Z") #t "2008-10-19" (t/offset-date-time "1970-01-01T10:23:54Z")] 43 | [3 "baz" (t/offset-date-time "2012-10-19T10:23:54Z") #t "2012-10-19" (t/offset-date-time "1970-01-01T10:23:54Z")]]) 44 | 45 | (def default-connection-params 46 | {:classname "com.clickhouse.jdbc.ClickHouseDriver" 47 | :subprotocol "clickhouse" 48 | :subname "//localhost:8123/default" 49 | :user "default" 50 | :password "" 51 | :ssl false 52 | :use_server_time_zone_for_dates true 53 | :product_name "metabase/1.53.4" 54 | :jdbc_ignore_unsupported_values "true" 55 | :jdbc_schema_term "schema", 56 | :max_open_connections 100 57 | :remember_last_set_roles true 58 | :http_connection_provider "HTTP_URL_CONNECTION" 59 | :custom_http_params ""}) 60 | 61 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Boolean] [_ _] "Boolean") 62 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/BigInteger] [_ _] "Int64") 63 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Char] [_ _] "String") 64 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Date] [_ _] "Date") 65 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/DateTime] [_ _] "DateTime64(3, 'GMT0')") 66 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/DateTimeWithLocalTZ] [_ _] "DateTime64(3, 'UTC')") 67 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Float] [_ _] "Float64") 68 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Integer] [_ _] "Int32") 69 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/IPAddress] [_ _] "IPv4") 70 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Text] [_ _] "String") 71 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/UUID] [_ _] "UUID") 72 | (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Time] [_ _] "Time") 73 | 74 | (defmethod tx/sorts-nil-first? :clickhouse [_ _] false) 75 | 76 | (defmethod tx/dbdef->connection-details :clickhouse [_ context {:keys [database-name]}] 77 | (merge 78 | {:host (tx/db-test-env-var-or-throw :clickhouse :host "localhost") 79 | :port (tx/db-test-env-var-or-throw :clickhouse :port 8123) 80 | :timezone :America/Los_Angeles} 81 | (when-let [user (tx/db-test-env-var :clickhouse :user)] 82 | {:user user}) 83 | (when-let [password (tx/db-test-env-var :clickhouse :password)] 84 | {:password password}) 85 | (when (= context :db) 86 | {:db database-name}))) 87 | 88 | (defmethod sql.tx/qualified-name-components :clickhouse 89 | ([_ db-name] [db-name]) 90 | ([_ db-name table-name] [db-name table-name]) 91 | ([_ db-name table-name field-name] [db-name table-name field-name])) 92 | 93 | (defmethod tx/create-db! :clickhouse 94 | [driver {:keys [database-name], :as db-def} & options] 95 | (let [database-name (ddl.i/format-name driver database-name)] 96 | (log/infof "Creating ClickHouse database %s" (pr-str database-name)) 97 | ;; call the default impl for SQL JDBC drivers 98 | (apply (get-method tx/create-db! :sql-jdbc/test-extensions) driver db-def options))) 99 | 100 | (defmethod ddl/insert-rows-dml-statements :clickhouse 101 | [driver table-identifier rows] 102 | (binding [driver/*compile-with-inline-parameters* true] 103 | ((get-method ddl/insert-rows-dml-statements :sql-jdbc/test-extensions) driver table-identifier rows))) 104 | 105 | (mu/defmethod load-data/do-insert! :clickhouse 106 | [driver :- :keyword 107 | ^java.sql.Connection conn :- (lib.schema.common/instance-of-class java.sql.Connection) 108 | table-identifier 109 | rows] 110 | (let [statements (ddl/insert-rows-dml-statements driver table-identifier rows)] 111 | (doseq [sql-args statements 112 | :let [sql-args (if (string? sql-args) 113 | [sql-args] 114 | sql-args)]] 115 | (assert (string? (first sql-args)) 116 | (format "Bad sql-args: %s" (pr-str sql-args))) 117 | (log/tracef "[insert] %s" (pr-str sql-args)) 118 | (try 119 | (jdbc/execute! {:connection conn :transaction? false} 120 | sql-args 121 | {:set-parameters (fn [stmt params] 122 | (sql-jdbc.execute/set-parameters! driver stmt params))}) 123 | (catch Throwable e 124 | (throw (ex-info (format "INSERT FAILED: %s" (ex-message e)) 125 | {:driver driver 126 | :sql-args (into [(str/split-lines (mdb.query/format-sql (first sql-args)))] 127 | (rest sql-args))} 128 | e))))))) 129 | 130 | (defn- quote-name 131 | [name] 132 | (sql.u/quote-name :clickhouse :field (ddl.i/format-name :clickhouse name))) 133 | 134 | (def ^:private non-nullable-types ["Array" "Map" "Tuple"]) 135 | (defn- disallowed-as-nullable? 136 | [ch-type] 137 | (boolean (some #(str/starts-with? ch-type %) non-nullable-types))) 138 | 139 | (defn- field->clickhouse-column 140 | [field] 141 | (let [{:keys [field-name base-type pk?]} field 142 | ch-type (if (map? base-type) 143 | (:native base-type) 144 | (sql.tx/field-base-type->sql-type :clickhouse base-type)) 145 | col-name (quote-name field-name) 146 | ch-col (cond 147 | (or pk? (disallowed-as-nullable? ch-type)) 148 | (format "%s %s" col-name ch-type) 149 | (= ch-type "Time") 150 | (format "%s Nullable(DateTime64) COMMENT 'time'" col-name) 151 | ; _ 152 | :else (format "%s Nullable(%s)" col-name ch-type))] 153 | ch-col)) 154 | 155 | (defn- ->comma-separated-str 156 | [coll] 157 | (->> coll 158 | (interpose ", ") 159 | (apply str))) 160 | 161 | (defmethod sql.tx/create-table-sql :clickhouse 162 | [_ {:keys [database-name]} {:keys [table-name field-definitions]}] 163 | (let [table-name (sql.tx/qualify-and-quote :clickhouse database-name table-name) 164 | pk-fields (filter (fn [{:keys [pk?]}] pk?) field-definitions) 165 | pk-field-names (map #(quote-name (:field-name %)) pk-fields) 166 | fields (->> field-definitions 167 | (map field->clickhouse-column) 168 | (->comma-separated-str)) 169 | order-by (->comma-separated-str pk-field-names)] 170 | (format "CREATE TABLE %s (%s) 171 | ENGINE = MergeTree 172 | ORDER BY (%s) 173 | SETTINGS allow_nullable_key=1" 174 | table-name fields order-by))) 175 | 176 | (defmethod execute/execute-sql! :clickhouse [& args] 177 | (apply execute/sequentially-execute-sql! args)) 178 | 179 | (defmethod load-data/row-xform :clickhouse [_driver _dbdef tabledef] 180 | (load-data/maybe-add-ids-xform tabledef)) 181 | 182 | (defmethod sql.tx/pk-sql-type :clickhouse [_] "Int32") 183 | 184 | (defmethod sql.tx/add-fk-sql :clickhouse [& _] nil) 185 | 186 | (defmethod sql.tx/session-schema :clickhouse [_] "default") 187 | 188 | (defn rows-without-index 189 | "Remove the Metabase index which is the first column in the result set" 190 | [query-result] 191 | (map #(drop 1 %) (qp.test/rows query-result))) 192 | 193 | (def ^:private test-db-initialized? (atom false)) 194 | (defn create-test-db! 195 | "Create a ClickHouse database called `metabase_test` and initialize some test data" 196 | [f] 197 | (when (not @test-db-initialized?) 198 | (let [details (tx/dbdef->connection-details :clickhouse :db {:database-name "metabase_test"})] 199 | ;; (println "### Executing create-test-db! with details:" details) 200 | (jdbc/with-db-connection 201 | [spec (sql-jdbc.conn/connection-details->spec :clickhouse (merge {:engine :clickhouse} details))] 202 | (let [raw-statements (slurp (io/resource "metabase/test/data/clickhouse_datasets.sql")) 203 | statements (as-> raw-statements s 204 | (str/split s #";") 205 | (map str/trim s) 206 | (filter seq s))] 207 | ;; (println "## Executing statements " statements) 208 | (jdbc/db-do-commands spec false statements) 209 | (reset! test-db-initialized? true))) 210 | ;; (println "### Done with executing create-test-db! with details:" details) 211 | )) 212 | (f)) 213 | 214 | #_{:clj-kondo/ignore [:warn-on-reflection]} 215 | (defn exec-statements 216 | ([statements details-map] 217 | (exec-statements statements details-map nil)) 218 | ([statements details-map clickhouse-settings] 219 | (sql-jdbc.execute/do-with-connection-with-options 220 | :clickhouse 221 | (sql-jdbc.conn/connection-details->spec :clickhouse (merge {:engine :clickhouse} details-map)) 222 | {:write? true} 223 | (fn [^java.sql.Connection conn] 224 | (doseq [statement statements] 225 | ;; (println "Executing:" statement) 226 | (let [^com.clickhouse.jdbc.ConnectionImpl clickhouse-conn (.unwrap conn com.clickhouse.jdbc.ConnectionImpl) 227 | query-settings (new com.clickhouse.client.api.query.QuerySettings)] 228 | (with-open [jdbcStmt (.createStatement conn)] 229 | (when clickhouse-settings 230 | (doseq [[k v] clickhouse-settings] (.setOption query-settings k v))) 231 | (.setDefaultQuerySettings clickhouse-conn query-settings) 232 | (.execute jdbcStmt statement)))))))) 233 | 234 | (defn do-with-test-db 235 | "Execute a test function using the test dataset" 236 | [f] 237 | (t2.with-temp/with-temp 238 | [:model/Database database 239 | {:engine :clickhouse 240 | :details (tx/dbdef->connection-details :clickhouse :db {:database-name "metabase_test"})}] 241 | (sync/sync-db-metadata! database) 242 | (f database))) 243 | 244 | (defmethod tx/dataset-already-loaded? :clickhouse 245 | [driver dbdef] 246 | (let [tabledef (first (:table-definitions dbdef)) 247 | db-name (ddl.i/format-name :clickhouse (:database-name dbdef)) 248 | table-name (ddl.i/format-name :clickhouse (:table-name tabledef)) 249 | details (tx/dbdef->connection-details :clickhouse :db {:database-name db-name})] 250 | (sql-jdbc.execute/do-with-connection-with-options 251 | driver 252 | (sql-jdbc.conn/connection-details->spec driver details) 253 | {:write? false} 254 | (fn [^java.sql.Connection conn] 255 | (with-open [rset (.getTables (.getMetaData conn) 256 | #_catalog nil 257 | #_schema-pattern db-name 258 | #_table-pattern table-name 259 | #_types (into-array String ["TABLE"]))] 260 | ;; if the ResultSet returns anything we know the table is already loaded. 261 | (.next rset)))))) 262 | -------------------------------------------------------------------------------- /src/metabase/driver/clickhouse.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.clickhouse 2 | "Driver for ClickHouse databases" 3 | #_{:clj-kondo/ignore [:unsorted-required-namespaces]} 4 | (:require [clojure.core.memoize :as memoize] 5 | [clojure.string :as str] 6 | [metabase.config :as config] 7 | [metabase.driver :as driver] 8 | [metabase.driver.clickhouse-introspection] 9 | [metabase.driver.clickhouse-nippy] 10 | [metabase.driver.clickhouse-qp] 11 | [metabase.driver.clickhouse-version :as clickhouse-version] 12 | [metabase.driver.ddl.interface :as ddl.i] 13 | [metabase.driver.sql :as driver.sql] 14 | [metabase.driver.sql-jdbc :as sql-jdbc] 15 | [metabase.driver.sql-jdbc.common :as sql-jdbc.common] 16 | [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] 17 | [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] 18 | [metabase.driver.sql.util :as sql.u] 19 | [metabase.lib.metadata :as lib.metadata] 20 | [metabase.query-processor.store :as qp.store] 21 | [metabase.upload :as upload] 22 | [metabase.util :as u] 23 | [metabase.util.log :as log]) 24 | (:import [com.clickhouse.client.api.query QuerySettings])) 25 | 26 | (set! *warn-on-reflection* true) 27 | 28 | (System/setProperty "clickhouse.jdbc.v2" "true") 29 | (driver/register! :clickhouse :parent #{:sql-jdbc}) 30 | 31 | (defmethod driver/display-name :clickhouse [_] "ClickHouse") 32 | (def ^:private product-name "metabase/1.53.4") 33 | 34 | (defmethod driver/prettify-native-form :clickhouse 35 | [_ native-form] 36 | (sql.u/format-sql-and-fix-params :mysql native-form)) 37 | 38 | (doseq [[feature supported?] {:standard-deviation-aggregations true 39 | :now true 40 | :set-timezone true 41 | :convert-timezone false 42 | :test/jvm-timezone-setting false 43 | :test/date-time-type false 44 | :test/time-type false 45 | :schemas true 46 | :datetime-diff true 47 | :upload-with-auto-pk false 48 | :window-functions/offset false 49 | :window-functions/cumulative (not config/is-test?) 50 | :left-join (not config/is-test?) 51 | :describe-fks false 52 | :actions false 53 | :metadata/key-constraints (not config/is-test?)}] 54 | (defmethod driver/database-supports? [:clickhouse feature] [_driver _feature _db] supported?)) 55 | 56 | (def ^:private default-connection-details 57 | {:user "default" :password "" :dbname "default" :host "localhost" :port "8123"}) 58 | 59 | (defn- connection-details->spec* [details] 60 | (let [;; ensure defaults merge on top of nils 61 | details (reduce-kv (fn [m k v] (assoc m k (or v (k default-connection-details)))) 62 | default-connection-details 63 | details) 64 | {:keys [user password dbname host port ssl clickhouse-settings max-open-connections]} details 65 | ;; if multiple databases were specified for the connection, 66 | ;; use only the first dbname as the "main" one 67 | dbname (first (str/split (str/trim dbname) #" ")) 68 | host (cond ; JDBCv1 used to accept schema in the `host` configuration option 69 | (str/starts-with? host "http://") (subs host 7) 70 | (str/starts-with? host "https://") (subs host 8) 71 | :else host)] 72 | (-> 73 | {:classname "com.clickhouse.jdbc.ClickHouseDriver" 74 | :subprotocol "clickhouse" 75 | :subname (str "//" host ":" port "/" dbname) 76 | :password (or password "") 77 | :user user 78 | :ssl (boolean ssl) 79 | :use_server_time_zone_for_dates true 80 | :product_name product-name 81 | :remember_last_set_roles true 82 | :http_connection_provider "HTTP_URL_CONNECTION" 83 | :jdbc_ignore_unsupported_values "true" 84 | :jdbc_schema_term "schema" 85 | :max_open_connections (or max-open-connections 100) 86 | ;; see also: https://clickhouse.com/docs/en/integrations/java#configuration 87 | :custom_http_params (or clickhouse-settings "")} 88 | (sql-jdbc.common/handle-additional-options details :separator-style :url)))) 89 | 90 | (defmethod sql-jdbc.execute/do-with-connection-with-options :clickhouse 91 | [driver db-or-id-or-spec {:keys [^String session-timezone _write?] :as options} f] 92 | (sql-jdbc.execute/do-with-resolved-connection 93 | driver 94 | db-or-id-or-spec 95 | options 96 | (fn [^java.sql.Connection conn] 97 | (when-not (sql-jdbc.execute/recursive-connection?) 98 | (when session-timezone 99 | (let [^com.clickhouse.jdbc.ConnectionImpl clickhouse-conn (.unwrap conn com.clickhouse.jdbc.ConnectionImpl) 100 | query-settings (new QuerySettings)] 101 | (.setOption query-settings "session_timezone" session-timezone) 102 | (.setDefaultQuerySettings clickhouse-conn query-settings))) 103 | (sql-jdbc.execute/set-best-transaction-level! driver conn) 104 | (sql-jdbc.execute/set-time-zone-if-supported! driver conn session-timezone) 105 | (when-let [db (cond 106 | ;; id? 107 | (integer? db-or-id-or-spec) 108 | (qp.store/with-metadata-provider db-or-id-or-spec 109 | (lib.metadata/database (qp.store/metadata-provider))) 110 | ;; db? 111 | (u/id db-or-id-or-spec) db-or-id-or-spec 112 | ;; otherwise it's a spec and we can't get the db 113 | :else nil)] 114 | (sql-jdbc.execute/set-role-if-supported! driver conn db))) 115 | (f conn)))) 116 | 117 | (def ^:private ^{:arglists '([db-details])} cloud? 118 | "Returns true if the `db-details` are for a ClickHouse Cloud instance, and false otherwise. If it fails to connect 119 | to the database, it throws a java.sql.SQLException." 120 | (memoize/ttl 121 | (fn [db-details] 122 | (let [spec (connection-details->spec* db-details)] 123 | (sql-jdbc.execute/do-with-connection-with-options 124 | :clickhouse spec nil 125 | (fn [^java.sql.Connection conn] 126 | (with-open [stmt (.createStatement conn) 127 | rset (.executeQuery stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")] 128 | (if (.next rset) (.getBoolean rset 1) false)))))) 129 | ;; cache the results for 48 hours; TTL is here only to eventually clear out old entries 130 | :ttl/threshold (* 48 60 60 1000))) 131 | 132 | (defmethod sql-jdbc.conn/connection-details->spec :clickhouse 133 | [_ details] 134 | (cond-> (connection-details->spec* details) 135 | (try (cloud? details) 136 | (catch java.sql.SQLException _e 137 | false)) 138 | ;; select_sequential_consistency guarantees that we can query data from any replica in CH Cloud 139 | ;; immediately after it is written 140 | (assoc :select_sequential_consistency true))) 141 | 142 | (defmethod driver/database-supports? [:clickhouse :uploads] [_driver _feature db] 143 | (if (:details db) 144 | (try (cloud? (:details db)) 145 | (catch java.sql.SQLException _e 146 | false)) 147 | false)) 148 | 149 | (defmethod driver/can-connect? :clickhouse 150 | [driver details] 151 | (if config/is-test? 152 | (try 153 | ;; Default SELECT 1 is not enough for Metabase test suite, 154 | ;; as it works slightly differently than expected there 155 | (let [spec (sql-jdbc.conn/connection-details->spec driver details) 156 | db (ddl.i/format-name driver (or (:dbname details) (:db details) "default"))] 157 | (sql-jdbc.execute/do-with-connection-with-options 158 | driver spec nil 159 | (fn [^java.sql.Connection conn] 160 | (let [stmt (.prepareStatement conn "SELECT count(*) > 0 FROM system.databases WHERE name = ?") 161 | _ (.setString stmt 1 db) 162 | rset (.executeQuery stmt)] 163 | (when (.next rset) 164 | (.getBoolean rset 1)))))) 165 | (catch Throwable e 166 | (log/error e "An exception during ClickHouse connectivity check") 167 | false)) 168 | ;; During normal usage, fall back to the default implementation 169 | (sql-jdbc.conn/can-connect? driver details))) 170 | 171 | (defmethod driver/db-default-timezone :clickhouse 172 | [driver database] 173 | (sql-jdbc.execute/do-with-connection-with-options 174 | driver database nil 175 | (fn [^java.sql.Connection conn] 176 | (with-open [stmt (.createStatement conn) 177 | rset (.executeQuery stmt "SELECT timezone() AS tz")] 178 | (when (.next rset) 179 | (.getString rset 1)))))) 180 | 181 | (defmethod driver/db-start-of-week :clickhouse [_] :monday) 182 | 183 | (defmethod ddl.i/format-name :clickhouse 184 | [_ table-or-field-name] 185 | (when table-or-field-name 186 | (str/replace table-or-field-name #"-" "_"))) 187 | 188 | ;;; ------------------------------------------ Connection Impersonation ------------------------------------------ 189 | 190 | (defmethod driver/upload-type->database-type :clickhouse 191 | [_driver upload-type] 192 | (case upload-type 193 | ::upload/varchar-255 "Nullable(String)" 194 | ::upload/text "Nullable(String)" 195 | ::upload/int "Nullable(Int64)" 196 | ::upload/float "Nullable(Float64)" 197 | ::upload/boolean "Nullable(Boolean)" 198 | ::upload/date "Nullable(Date32)" 199 | ::upload/datetime "Nullable(DateTime64(3))" 200 | ::upload/offset-datetime nil)) 201 | 202 | (defmethod driver/table-name-length-limit :clickhouse 203 | [_driver] 204 | ;; FIXME: This is a lie because you're really limited by a filesystems' limits, because Clickhouse uses 205 | ;; filenames as table/column names. But its an approximation 206 | 206) 207 | 208 | (defn- quote-name [s] 209 | (let [parts (str/split (name s) #"\.")] 210 | (str/join "." (map #(str "`" % "`") parts)))) 211 | 212 | (defn- create-table!-sql 213 | "Creates a ClickHouse table with the given name and column definitions. It assumes the engine is MergeTree, 214 | so it only works with Clickhouse Cloud and single node on-premise deployments at the moment." 215 | [_driver table-name column-definitions & {:keys [primary-key] :as opts}] 216 | (str/join "\n" 217 | [(#'sql-jdbc/create-table!-sql :sql-jdbc table-name column-definitions opts) 218 | "ENGINE = MergeTree" 219 | (format "ORDER BY (%s)" (str/join ", " (map quote-name primary-key))) 220 | ;; disable insert idempotency to allow duplicate inserts 221 | "SETTINGS replicated_deduplication_window = 0"])) 222 | 223 | (defmethod driver/create-table! :clickhouse 224 | [driver db-id table-name column-definitions & {:keys [primary-key]}] 225 | (sql-jdbc.execute/do-with-connection-with-options 226 | driver 227 | db-id 228 | {:write? true} 229 | (fn [^java.sql.Connection conn] 230 | (with-open [stmt (.createStatement conn)] 231 | (.execute stmt (create-table!-sql driver table-name column-definitions :primary-key primary-key)))))) 232 | 233 | (defmethod driver/insert-into! :clickhouse 234 | [driver db-id table-name column-names values] 235 | (when (seq values) 236 | (sql-jdbc.execute/do-with-connection-with-options 237 | driver 238 | db-id 239 | {:write? true} 240 | (fn [^java.sql.Connection conn] 241 | (let [sql (format "INSERT INTO %s (%s) VALUES (%s)" 242 | (quote-name table-name) 243 | (str/join ", " (map quote-name column-names)) 244 | (str/join ", " (repeat (count column-names) "?")))] 245 | (with-open [ps (.prepareStatement conn sql)] 246 | (doseq [row values] 247 | (when (seq row) 248 | (doseq [[idx v] (map-indexed (fn [x y] [(inc x) y]) row)] 249 | (condp isa? (type v) 250 | nil (.setString ps idx nil) 251 | java.lang.String (.setString ps idx v) 252 | java.lang.Boolean (.setBoolean ps idx v) 253 | java.lang.Long (.setLong ps idx v) 254 | java.lang.Double (.setFloat ps idx v) 255 | java.math.BigInteger (.setObject ps idx v) 256 | java.time.LocalDate (.setObject ps idx v) 257 | java.time.LocalDateTime (.setObject ps idx v) 258 | java.time.OffsetDateTime (.setObject ps idx v) 259 | (.setString ps idx (str v)))) 260 | (.addBatch ps))) 261 | (doall (.executeBatch ps)))))))) 262 | 263 | ;;; ------------------------------------------ User Impersonation ------------------------------------------ 264 | 265 | (defmethod driver/database-supports? [:clickhouse :connection-impersonation] 266 | [_driver _feature db] 267 | (if db 268 | (try (clickhouse-version/is-at-least? 24 4 db) 269 | (catch Throwable _e 270 | false)) 271 | false)) 272 | 273 | (defmethod driver.sql/set-role-statement :clickhouse 274 | [_ role] 275 | (let [default-role (driver.sql/default-database-role :clickhouse nil) 276 | quote-if-needed (fn [r] 277 | (if (or (re-matches #"\".*\"" r) (= role default-role)) 278 | r 279 | (format "\"%s\"" r))) 280 | quoted-role (->> (str/split role #",") 281 | (map quote-if-needed) 282 | (str/join ",")) 283 | statement (format "SET ROLE %s" quoted-role)] 284 | statement)) 285 | 286 | (defmethod driver.sql/default-database-role :clickhouse 287 | [_ _] 288 | "NONE") 289 | -------------------------------------------------------------------------------- /test/metabase/driver/clickhouse_substitution_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.clickhouse-substitution-test 2 | #_{:clj-kondo/ignore [:unsorted-required-namespaces]} 3 | (:require [clojure.test :refer :all] 4 | [java-time.api :as t] 5 | [metabase.query-processor :as qp] 6 | [metabase.test :as mt] 7 | [metabase.test.data :as data] 8 | [metabase.test.data.interface :as tx] 9 | [metabase.test.data.clickhouse :as ctd] 10 | [metabase.util :as u] 11 | [schema.core :as s]) 12 | (:import (java.time LocalDate LocalDateTime))) 13 | 14 | (set! *warn-on-reflection* true) 15 | 16 | (defn- get-mbql 17 | [value db] 18 | (let [uuid (str (java.util.UUID/randomUUID))] 19 | {:database (mt/id) 20 | :type "native" 21 | :native {:collection "test-table" 22 | :template-tags 23 | {:x {:id uuid 24 | :name "d" 25 | :display-name "D" 26 | :type "dimension" 27 | :dimension ["field" (mt/id :test-table :d) nil] 28 | :required true}} 29 | :query (format "SELECT * FROM `%s`.`test_table` WHERE {{x}}" db)} 30 | :parameters [{:type "date/all-options" 31 | :value value 32 | :target ["dimension" ["template-tag" "x"]] 33 | :id uuid}]})) 34 | 35 | (def ^:private clock (t/mock-clock (t/instant "2019-11-30T23:00:00Z") (t/zone-id "UTC"))) 36 | (s/defn ^:private local-date-now :- LocalDate [] (LocalDate/now clock)) 37 | (s/defn ^:private local-date-time-now :- LocalDateTime [] (LocalDateTime/now clock)) 38 | 39 | (deftest ^:parallel clickhouse-variables-field-filters-datetime-and-datetime64 40 | (mt/test-driver 41 | :clickhouse 42 | (mt/with-clock clock 43 | (letfn 44 | [(->clickhouse-input 45 | [^LocalDateTime ldt] 46 | [(t/format "yyyy-MM-dd HH:mm:ss" ldt)]) 47 | (get-test-table 48 | [rows native-type] 49 | ["test_table" 50 | [{:field-name "d" 51 | :base-type {:native native-type}}] 52 | (map ->clickhouse-input rows)]) 53 | (->iso-str 54 | [^LocalDateTime ldt] 55 | (t/format "yyyy-MM-dd'T'HH:mm:ss'Z'" ldt))] 56 | (doseq [base-type ["DateTime" "DateTime64"]] 57 | (testing base-type 58 | (testing "on specific" 59 | (let [db (format "mb_vars_on_x_%s" 60 | (u/lower-case-en base-type)) 61 | now (local-date-time-now) 62 | row1 (.minusHours now 14) 63 | row2 (.minusMinutes now 20) 64 | row3 (.plusMinutes now 5) 65 | row4 (.plusHours now 6) 66 | table (get-test-table [row1 row2 row3 row4] base-type)] 67 | (data/dataset 68 | (tx/dataset-definition db table) 69 | (testing "date" 70 | (is (= [[(->iso-str row1)] [(->iso-str row2)] [(->iso-str row3)]] 71 | (ctd/rows-without-index (qp/process-query (get-mbql "2019-11-30" db)))))) 72 | (testing "datetime" 73 | (is (= [[(->iso-str row2)]] 74 | (ctd/rows-without-index (qp/process-query (get-mbql "2019-11-30T22:40:00" db))))))))) 75 | (testing "past/next minutes" 76 | (let [db (format "mb_vars_m_%s" 77 | (u/lower-case-en base-type)) 78 | now (local-date-time-now) 79 | row1 (.minusHours now 14) 80 | row2 (.minusMinutes now 20) 81 | row3 (.plusMinutes now 5) 82 | row4 (.plusHours now 6) 83 | table (get-test-table [row1 row2 row3 row4] base-type)] 84 | (data/dataset 85 | (tx/dataset-definition db table) 86 | (testing "past30minutes" 87 | (is (= [[(->iso-str row2)]] 88 | (ctd/rows-without-index (qp/process-query (get-mbql "past30minutes" db)))))) 89 | (testing "next30minutes" 90 | (is (= [[(->iso-str row3)]] 91 | (ctd/rows-without-index (qp/process-query (get-mbql "next30minutes" db))))))))) 92 | (testing "past/next hours" 93 | (let [db (format "mb_vars__past_next_hours_%s" 94 | (u/lower-case-en base-type)) 95 | now (local-date-time-now) 96 | row1 (.minusHours now 14) 97 | row2 (.minusHours now 2) 98 | row3 (.plusHours now 25) 99 | row4 (.plusHours now 6) 100 | table (get-test-table [row1 row2 row3 row4] base-type)] 101 | (data/dataset 102 | (tx/dataset-definition db table) 103 | (testing "past12hours" 104 | (is (= [[(->iso-str row2)]] 105 | (ctd/rows-without-index (qp/process-query (get-mbql "past12hours" db)))))) 106 | (testing "next12hours" 107 | (is (= [[(->iso-str row4)]] 108 | (ctd/rows-without-index (qp/process-query (get-mbql "next12hours" db))))))))) 109 | (testing "past/next days" 110 | (let [db (format "mb_vars_d_%s" 111 | (u/lower-case-en base-type)) 112 | now (local-date-time-now) 113 | row1 (.minusDays now 14) 114 | row2 (.minusDays now 2) 115 | row3 (.plusDays now 25) 116 | row4 (.plusDays now 6) 117 | table (get-test-table [row1 row2 row3 row4] base-type)] 118 | (data/dataset 119 | (tx/dataset-definition db table) 120 | (testing "past12days" 121 | (is (= [[(->iso-str row2)]] 122 | (ctd/rows-without-index (qp/process-query (get-mbql "past12days" db)))))) 123 | (testing "next12days" 124 | (is (= [[(->iso-str row4)]] 125 | (ctd/rows-without-index (qp/process-query (get-mbql "next12days" db))))))))) 126 | (testing "past/next months/quarters" 127 | (let [db (format "mb_vars_m_q_%s" 128 | (u/lower-case-en base-type)) 129 | now (local-date-time-now) 130 | row1 (.minusMonths now 14) 131 | row2 (.minusMonths now 4) 132 | row3 (.plusMonths now 25) 133 | row4 (.plusMonths now 6) 134 | table (get-test-table [row1 row2 row3 row4] base-type)] 135 | (data/dataset 136 | (tx/dataset-definition db table) 137 | (testing "past12months" 138 | (is (= [[(->iso-str row2)]] 139 | (ctd/rows-without-index (qp/process-query (get-mbql "past12months" db)))))) 140 | (testing "next12months" 141 | (is (= [[(->iso-str row4)]] 142 | (ctd/rows-without-index (qp/process-query (get-mbql "next12months" db)))))) 143 | (testing "past3quarters" 144 | (is (= [[(->iso-str row2)]] 145 | (ctd/rows-without-index (qp/process-query (get-mbql "past3quarters" db)))))) 146 | (testing "next3quarters" 147 | (is (= [[(->iso-str row4)]] 148 | (ctd/rows-without-index (qp/process-query (get-mbql "next3quarters" db))))))))) 149 | (testing "past/next years" 150 | (let [db (format "mb_vars_y_%s" 151 | (u/lower-case-en base-type)) 152 | now (local-date-time-now) 153 | row1 (.minusYears now 14) 154 | row2 (.minusYears now 4) 155 | row3 (.plusYears now 25) 156 | row4 (.plusYears now 6) 157 | table (get-test-table [row1 row2 row3 row4] base-type)] 158 | (data/dataset 159 | (tx/dataset-definition db table) 160 | (testing "past12years" 161 | (is (= [[(->iso-str row2)]] 162 | (ctd/rows-without-index (qp/process-query (get-mbql "past12years" db)))))) 163 | (testing "next12years" 164 | (is (= [[(->iso-str row4)]] 165 | (ctd/rows-without-index (qp/process-query (get-mbql "next12years" db))))))))))))))) 166 | 167 | (deftest ^:parallel clickhouse-variables-field-filters-date-and-date32 168 | (mt/test-driver 169 | :clickhouse 170 | (mt/with-clock clock 171 | (letfn 172 | [(->clickhouse-input 173 | [^LocalDate ld] 174 | [(t/format "yyyy-MM-dd" ld)]) 175 | (get-test-table 176 | [rows native-type] 177 | ["test_table" 178 | [{:field-name "d" 179 | :base-type {:native native-type}}] 180 | (map ->clickhouse-input rows)]) 181 | (->iso-str 182 | [^LocalDate ld] 183 | (str (t/format "yyyy-MM-dd" ld) "T00:00:00Z"))] 184 | (doseq [base-type ["Date" "Date32"]] 185 | (testing base-type 186 | (testing "on specific date" 187 | (let [db (format "mb_vars_on_x_%s" 188 | (u/lower-case-en base-type)) 189 | now (local-date-time-now) 190 | row1 (.minusDays now 14) 191 | row2 now 192 | row3 (.plusDays now 25) 193 | row4 (.plusDays now 6) 194 | table (get-test-table [row1 row2 row3 row4] base-type)] 195 | (data/dataset 196 | (tx/dataset-definition db table) 197 | (is (= [[(->iso-str row2)]] 198 | (ctd/rows-without-index (qp/process-query (get-mbql "2019-11-30" db)))))))) 199 | (testing "past/next days" 200 | (let [db (format "mb_vars_d_%s" 201 | (u/lower-case-en base-type)) 202 | now (local-date-now) 203 | row1 (.minusDays now 14) 204 | row2 (.minusDays now 2) 205 | row3 (.plusDays now 25) 206 | row4 (.plusDays now 6) 207 | table (get-test-table [row1 row2 row3 row4] base-type)] 208 | (data/dataset 209 | (tx/dataset-definition db table) 210 | (testing "past12days" 211 | (is (= [[(->iso-str row2)]] 212 | (ctd/rows-without-index (qp/process-query (get-mbql "past12days" db)))))) 213 | (testing "next12days" 214 | (is (= [[(->iso-str row4)]] 215 | (ctd/rows-without-index (qp/process-query (get-mbql "next12days" db))))))))) 216 | (testing "past/next months/quarters" 217 | (let [db (format "mb_vars_m_q_%s" 218 | (u/lower-case-en base-type)) 219 | now (local-date-now) 220 | row1 (.minusMonths now 14) 221 | row2 (.minusMonths now 4) 222 | row3 (.plusMonths now 25) 223 | row4 (.plusMonths now 6) 224 | table (get-test-table [row1 row2 row3 row4] base-type)] 225 | (data/dataset 226 | (tx/dataset-definition db table) 227 | (testing "past12months" 228 | (is (= [[(->iso-str row2)]] 229 | (ctd/rows-without-index (qp/process-query (get-mbql "past12months" db)))))) 230 | (testing "next12months" 231 | (is (= [[(->iso-str row4)]] 232 | (ctd/rows-without-index (qp/process-query (get-mbql "next12months" db)))))) 233 | (testing "past3quarters" 234 | (is (= [[(->iso-str row2)]] 235 | (ctd/rows-without-index (qp/process-query (get-mbql "past3quarters" db)))))) 236 | (testing "next3quarters" 237 | (is (= [[(->iso-str row4)]] 238 | (ctd/rows-without-index (qp/process-query (get-mbql "next3quarters" db))))))))) 239 | (testing "past/next years" 240 | (let [db (format "mb_vars_y_%s" 241 | (u/lower-case-en base-type)) 242 | now (local-date-now) 243 | row1 (.minusYears now 14) 244 | row2 (.minusYears now 4) 245 | row3 (.plusYears now 25) 246 | row4 (.plusYears now 6) 247 | table (get-test-table [row1 row2 row3 row4] base-type)] 248 | (data/dataset 249 | (tx/dataset-definition db table) 250 | (testing "past12years" 251 | (is (= [[(->iso-str row2)]] 252 | (ctd/rows-without-index (qp/process-query (get-mbql "past12years" db)))))) 253 | (testing "next12years" 254 | (is (= [[(->iso-str row4)]] 255 | (ctd/rows-without-index (qp/process-query (get-mbql "next12years" db))))))))))))))) 256 | 257 | (deftest ^:parallel clickhouse-variables-field-filters-null-dates 258 | (mt/test-driver 259 | :clickhouse 260 | (mt/with-clock clock 261 | (letfn 262 | [(->input-ld 263 | [^LocalDate ld] 264 | [(t/format "yyyy-MM-dd" ld)]) 265 | (->input-ldt 266 | [^LocalDateTime ldt] 267 | [(t/format "yyyy-MM-dd HH:mm:ss" ldt)]) 268 | (->iso-str-ld 269 | [^LocalDate ld] 270 | (str (t/format "yyyy-MM-dd" ld) "T00:00:00Z")) 271 | (->iso-str-ldt 272 | [^LocalDateTime ldt] 273 | (t/format "yyyy-MM-dd'T'HH:mm:ss'Z'" ldt))] 274 | (let [db "mb_vars_null_dates" 275 | now-ld (local-date-now) 276 | now-ldt (local-date-time-now) 277 | table ["test_table" 278 | [{:field-name "d" 279 | :base-type {:native "Date"}} 280 | {:field-name "d32" 281 | :base-type {:native "Date32"}} 282 | {:field-name "dt" 283 | :base-type {:native "DateTime"}} 284 | {:field-name "dt64" 285 | :base-type {:native "DateTime64"}}] 286 | [;; row 1 287 | [(->input-ld now-ld) nil (->input-ldt now-ldt) nil] 288 | ;; row 2 289 | [nil (->input-ld now-ld) nil (->input-ldt now-ldt)]]] 290 | first-row [[(->iso-str-ld now-ld) nil (->iso-str-ldt now-ldt) nil]] 291 | second-row [[nil (->iso-str-ld now-ld) nil (->iso-str-ldt now-ldt)]]] 292 | (data/dataset 293 | (tx/dataset-definition db table) 294 | (letfn 295 | [(get-mbql* 296 | [field value] 297 | (let [uuid (str (java.util.UUID/randomUUID))] 298 | {:database (mt/id) 299 | :type "native" 300 | :native {:collection "test-table" 301 | :template-tags 302 | {:x {:id uuid 303 | :name (str field) 304 | :display-name (str field) 305 | :type "dimension" 306 | :dimension ["field" (mt/id :test-table field) nil] 307 | :required true}} 308 | :query (format "SELECT * FROM `%s`.`test_table` WHERE {{x}}" db)} 309 | :parameters [{:type "date/all-options" 310 | :value value 311 | :target ["dimension" ["template-tag" "x"]] 312 | :id uuid}]}))] 313 | (testing "first row (Date field match)" 314 | (is (= first-row (ctd/rows-without-index (qp/process-query (get-mbql* :d "2019-11-30")))))) 315 | (testing "first row (DateTime field match)" 316 | (is (= first-row (ctd/rows-without-index (qp/process-query (get-mbql* :dt "2019-11-30T23:00:00")))))) 317 | (testing "second row (Date32 field match)" 318 | (is (= second-row (ctd/rows-without-index (qp/process-query (get-mbql* :d32 "2019-11-30")))))) 319 | (testing "second row (DateTime64 field match)" 320 | (is (= second-row (ctd/rows-without-index (qp/process-query (get-mbql* :dt64 "2019-11-30T23:00:00"))))))))))))) 321 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 1.53.4 2 | 3 | ### Improvements 4 | 5 | * The JDBC driver was updated to [0.8.4](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.4). This fixes [#309](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/309), [#300](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/300), [#297](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/297). 6 | 7 | # 1.53.3 8 | 9 | ### Improvements 10 | 11 | * If ClickHouse instance hostname was specified including `http://` or `https://` schema (e.g. `https://sub.example.com`), it will be automatically handled and removed by the driver, instead of failing with a connection error. 12 | * The JDBC driver was updated to [0.8.2](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.2) 13 | 14 | # 1.53.2 15 | 16 | ### Bug fixes 17 | 18 | * The JDBC driver was updated to [0.8.1](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.1) to fix errors in queries with CTEs ([#297](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/297), [#288](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/288), [tadeboro](https://github.com/tadeboro)). 19 | 20 | # 1.53.1 21 | 22 | ### Bug fixes 23 | 24 | * Fix unsigned integers overflow ([#293](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/293)) 25 | 26 | # 1.53.0 27 | 28 | ### Improvements 29 | 30 | * Adds Metabase 0.53.x support ([#287](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/287), [dpsutton](https://github.com/dpsutton)). 31 | 32 | ### Bug fixes 33 | 34 | * Fixed OOB exception on CSV insert caused by an incompatibility with JDBC v2 ([#286](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/286), [wotbrew](https://github.com/wotbrew)). 35 | 36 | # 1.52.0 37 | 38 | - Formal Metabase 0.52.x+ support 39 | - The driver now uses JDBC v2 ([0.8.0](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.0)) 40 | - Various improvements to handling of datetimes with timezones 41 | - `:convert-timezone` feature is disabled for now. 42 | - Added `max-open-connections` setting under "advanced options"; default is 100. 43 | 44 | # 1.51.0 45 | 46 | Adds Metabase 0.51.x support. 47 | 48 | # 1.50.7 49 | 50 | ### Improvements 51 | 52 | * Added a configuration field (under the "advanced options", hidden by default) to override certain ClickHouse settings ([#272](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/272)). 53 | 54 | # 1.50.6 55 | 56 | ### Bug fixes 57 | 58 | * Fixed null pointer exception on CSV insert ([#268](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/268), [crisptrutski](https://github.com/crisptrutski)). 59 | 60 | # 1.50.5 61 | 62 | ### Bug fixes 63 | 64 | * Fixed an error that could occur while setting roles containing hyphens for connection impersonation ([#266](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/266), [sharankow](https://github.com/sharankow)). 65 | 66 | # 1.50.4 67 | 68 | ### Bug fixes 69 | 70 | * Fixed an error while uploading a CSV with an offset datetime column ([#263](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/263), [crisptrutski](https://github.com/crisptrutski)). 71 | 72 | # 1.50.3 73 | 74 | ### Improvements 75 | 76 | * The driver no longer explicitly sets `allow_experimental_analyzer=0` settings on the connection level; the [new ClickHouse analyzer](https://clickhouse.com/docs/en/operations/analyzer) is now enabled by default. 77 | 78 | # 1.50.2 79 | 80 | ### Bug fixes 81 | 82 | * Fixed Array inner type introspection, which could cause reduced performance when querying tables containing arrays. ([#257](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/257)) 83 | 84 | # 1.50.1 85 | 86 | ### New features 87 | 88 | * Enabled `:set-timezone` ([docs](https://www.metabase.com/docs/latest/configuring-metabase/localization#report-timezone)) Metabase feature in the driver. ([#200](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/200)) 89 | * Enabled `:convert-timezone` ([docs](https://www.metabase.com/docs/latest/questions/query-builder/expressions/converttimezone)) Metabase feature in the driver. ([#254](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/254)) 90 | 91 | ### Other 92 | 93 | * The driver now uses [`session_timezone` ClickHouse setting](https://clickhouse.com/docs/en/operations/settings/settings#session_timezone). This is necessary to support the `:set-timezone` and `:convert-timezone` features; however, this setting [was introduced in 23.6](https://clickhouse.com/docs/en/whats-new/changelog/2023#236), which makes it the minimal required ClickHouse version to work with the driver. 94 | 95 | # 1.50.0 96 | 97 | After Metabase 0.50.0, a new naming convention exists for the driver's releases. The new one is intended to reflect the Metabase version the driver is supposed to run on. For example, the driver version 1.50.0 means that it should be used with Metabase v0.50.x or Metabase EE 1.50.x _only_, and it is _not guaranteed_ that this particular version of the driver can work with the previous or the following versions of Metabase. 98 | 99 | ### New features 100 | 101 | * Added Metabase 0.50.x support. 102 | 103 | ### Improvements 104 | 105 | * Bumped the JDBC driver to [0.6.1](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.6.1). 106 | 107 | ### Bug fixes 108 | 109 | * Fixed the issue where the connection impersonation feature support could be incorrectly reported as disabled. 110 | 111 | ### Other 112 | 113 | * The new ClickHouse analyzer, [which is enabled by default in 24.3+](https://clickhouse.com/blog/clickhouse-release-24-03#analyzer-enabled-by-default), is disabled for the queries executed by the driver, as it shows some compatibilities with the queries generated by Metabase (see [this issue](https://github.com/ClickHouse/ClickHouse/issues/64487) for more details). 114 | * The `:window-functions/offset` Metabase feature is currently disabled, as the default implementation generates queries incompatible with ClickHouse. See [this issue](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/245) for tracking. 115 | 116 | # 1.5.1 117 | 118 | Metabase 0.49.14+ only. 119 | 120 | ### Bug fixes 121 | 122 | * Fixed the issue where the Metabase instance could end up broken if the ClickHouse instance was _stopped_ during the upgrade to 1.5.0. ([#242](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/242)) 123 | * Fixed variables substitution with Nullable Date, Date32, DateTime, and DateTime64 columns, where the generated query could fail with NULL values in the database due to an incorrect cast call. ([#243](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/243)) 124 | 125 | # 1.5.0 126 | 127 | Metabase 0.49.14+ only. 128 | 129 | ### New features 130 | 131 | * Added [Metabase CSV Uploads feature](https://www.metabase.com/docs/latest/databases/uploads) support, which is currently enabled with ClickHouse Cloud only. On-premise deployments support will be added in the next release. ([calherries](https://github.com/calherries), [#236](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/236), [#238](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/238)) 132 | * Added [Metabase connection impersonation feature](https://www.metabase.com/learn/permissions/impersonation) support. This feature will be enabled by the driver only if ClickHouse version 24.4+ is detected. ([#219](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/219)) 133 | 134 | ### Improvements 135 | 136 | * Proper role setting support on cluster deployments (related issue: [#192](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/192)). 137 | * Bump the JDBC driver to [0.6.0-patch5](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.6.0-patch5). 138 | 139 | ### Bug fixes 140 | 141 | * Fixed missing data for the last day when using filters with DateTime columns. ([#202](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/202), [#229](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/229)) 142 | 143 | # 1.4.0 144 | 145 | ### New features 146 | * Metabase 0.49.x support. 147 | 148 | ### Bug fixes 149 | * Fixed an incorrect substitution for the current day filter with DateTime columns. ([#216](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/216)) 150 | 151 | # 1.3.4 152 | 153 | ### New features 154 | 155 | * If introspected ClickHouse version is lower than 23.8, the driver will not use [startsWithUTF8](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswithutf8) and fall back to its [non-UTF8 counterpart](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswith) instead. There is a drawback in this compatibility mode: potentially incorrect filtering results when working with non-latin strings. If your use case includes filtering by columns with such strings and you experience these issues, consider upgrading your ClickHouse server to 23.8+. ([#224](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/224)) 156 | 157 | # 1.3.3 158 | 159 | ### Bug fixes 160 | * Fixed an issue where it was not possible to create a connection with multiple databases using TLS. ([#215](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/215)) 161 | 162 | # 1.3.2 163 | 164 | ### Bug fixes 165 | * Remove `can-connect?` method override which could cause issues with editing or creating new connections. ([#212](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/212)) 166 | 167 | 168 | # 1.3.1 169 | 170 | ### Bug fixes 171 | * Fixed incorrect serialization of `Array(UInt8)` columns ([#209](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/209)) 172 | 173 | # 1.3.0 174 | 175 | ### New features 176 | * Metabase 0.48.x support 177 | 178 | ### Bug fixes 179 | * Fixed last/next minutes/hours filters with variables creating incorrect queries due to unnecessary `CAST col AS date` call. 180 | 181 | # 1.2.5 182 | 183 | Metabase 0.47.7+ only. 184 | 185 | ### New features 186 | * Added [datetimeDiff](https://www.metabase.com/docs/latest/questions/query-builder/expressions/datetimediff) function support ([#117](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/117)) 187 | 188 | # 1.2.4 189 | 190 | Metabase 0.47.7+ only. 191 | 192 | ### Bug fixes 193 | * Fixed UI question -> SQL conversion creating incorrect queries due to superfluous spaces in columns/tables/database names. 194 | 195 | # 1.2.3 196 | 197 | ### Bug fixes 198 | 199 | * Fixed `LowCardinality(Nullable)` types introspection, where it was incorrectly reported as `type/*` ([#203](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/203)) 200 | 201 | # 1.2.2 202 | 203 | ### Bug fixes 204 | * Removed forward slash from serialized IPv4/IPv6 columns. NB: IPv4/IPv6 columns are temporarily resolved as `type/TextLike` instead of `type/IPAddress` base type due to an unexpected result in Metabase 0.47 type check. 205 | * Removed superfluous CAST calls from generated queries that use Date* columns and/or intervals 206 | 207 | # 1.2.1 208 | ### New features 209 | * Use HoneySQL2 in the driver 210 | 211 | # 1.2.0 212 | 213 | ### New features 214 | * Metabase 0.47 support 215 | * Connection impersonation support (0.47 feature) 216 | 217 | ### Bug fixes 218 | * More correct general database type -> base type mapping 219 | * `DateTime64` is now correctly mapped to `:type/DateTime` 220 | * `database-required` field property is now correctly set to `true` if a field is not `Nullable` 221 | 222 | # 1.1.7 223 | 224 | ### New features 225 | 226 | * JDBC driver upgrade (v0.4.1 -> [v0.4.6](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.4.6)) 227 | * Support DateTime64 by [@lucas-tubi](https://github.com/lucas-tubi) ([#165](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/165)) 228 | * Use native `startsWith`/`endsWith` instead of `LIKE str%`/`LIKE %str` 229 | 230 | # 1.1.6 231 | 232 | ### Bug fixes 233 | 234 | * Fixed temporal bucketing issues (see [#155](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/155)) 235 | 236 | # 1.1.5 237 | 238 | ### Bug fixes 239 | 240 | * Fixed Nippy error on cached questions (see [#147](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/147)) 241 | 242 | # 1.1.4 243 | 244 | ### Bug fixes 245 | 246 | * Fixed `sum-where` behavior where previously it could not be applied to Int columns (see [#156](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/156)) 247 | 248 | # 1.1.3 249 | 250 | ### New features 251 | 252 | * Hide `.inner` tables of Materialized Views. 253 | * Resolve `Map` base type to `type/Dictionary`. 254 | * Database name can now contain multiple schemas in the UI field (space-separated by default), which tells the driver to scan selected databases. Separator can be set in `metabase.driver.clickhouse/SEPARATOR`. (@veschin) 255 | 256 | # 1.1.2 257 | 258 | ### Bug fixes 259 | 260 | * Now the driver is able to scan and work with `SimpleAggregateFunction` columns: those were excluded by mistake in 1.0.2. 261 | 262 | 263 | # 1.1.1 264 | 265 | ### New features 266 | 267 | * Metabase 0.46.x compatibility. 268 | * Added [cljc.java-time](https://clojars.org/com.widdindustries/cljc.java-time) to dependencies, as it is no longer loaded by Metabase. 269 | 270 | # 1.1.0 271 | 272 | ### New features 273 | 274 | * Update JDBC driver to v0.4.1. 275 | * Use new `product_name` additional option instead of `client_name` 276 | * Replace `sql-jdbc.execute/read-column [:clickhouse Types/ARRAY]` with `sql-jdbc.execute/read-column-thunk [:clickhouse Types/ARRAY]` to be compatible with Metabase 0.46 breaking changes once it is released. 277 | 278 | ### Bug fixes 279 | 280 | * Fix `sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIME]` return type. 281 | 282 | # 1.0.4 283 | 284 | ### New features 285 | 286 | * Adds a new "Scan all databases" UI toggle (disabled by default), which tells the driver to scan all available databases (excluding `system` and `information_schema`) instead of only the database it is connected to. 287 | * Database input moved below host/port/username/password in the UI. 288 | 289 | # 1.0.3 290 | 291 | ### Bug fixes 292 | 293 | * Fixed NPE that could be thrown by the driver in case of empty database name input. 294 | 295 | # 1.0.2 296 | 297 | ### Bug fixes 298 | 299 | * As the underlying JDBC driver version does not support columns with `(Simple)AggregationFunction` type, these columns are now excluded from the table metadata and data browser result sets to prevent sync or data browsing errors. 300 | 301 | # 1.0.1 302 | 303 | ### Bug fixes 304 | 305 | * Boolean base type inference fix by [@s-huk](https://github.com/s-huk) (see [#134](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/134)) 306 | 307 | # 1.0.0 308 | 309 | Formal stable release milestone. 310 | 311 | ### New features 312 | 313 | * Added HTTP User-Agent (via clickhouse-jdbc `client_name` setting) with the plugin info according to the [language client spec](https://docs.google.com/document/d/1924Dvy79KXIhfqKpi1EBVY3133pIdoMwgCQtZ-uhEKs/edit#heading=h.ah33hoz5xei2) 314 | 315 | # 0.9.2 316 | 317 | ### New features 318 | 319 | * Allow to bypass system-wide proxy settings [#120](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/120) 320 | 321 | It's the first plugin release from the ClickHouse organization. 322 | 323 | From now on, the plugin is distributed under the Apache 2.0 License. 324 | 325 | # 0.9.1 326 | 327 | ### New features 328 | 329 | * Metabase 0.45.x compatibility [#107](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/107) 330 | * Added SSH tunnel option [#116](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/116) 331 | 332 | # 0.9.0 333 | 334 | ### New features 335 | 336 | * Using https://github.com/ClickHouse/clickhouse-jdbc `v0.3.2-patch11` 337 | 338 | ### Bug fixes 339 | 340 | * URLs with underscores [#23](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/23) 341 | * `now()` timezones issues [#81](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/81) 342 | * Boolean errors [#88](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/88) 343 | 344 | NB: there are messages like this in the Metabase logs 345 | 346 | ``` 347 | 2022-12-07 11:20:58,056 WARN internal.ClickHouseConnectionImpl :: [JDBC Compliant Mode] Transaction is not supported. You may change jdbcCompliant to false to throw SQLException instead. 348 | 2022-12-07 11:20:58,056 WARN internal.ClickHouseConnectionImpl :: [JDBC Compliant Mode] Transaction [ce0e121a-419a-4414-ac39-30f79eff7afd] (0 queries & 0 savepoints) is committed. 349 | ``` 350 | 351 | Unfortunately, this is the behaviour of the underlying JDBC driver now. 352 | 353 | Please consider raising the log level for `com.clickhouse.jdbc.internal.ClickHouseConnectionImpl` to `ERROR`. 354 | 355 | # 0.8.3 356 | 357 | ### New features 358 | 359 | * Enable additional options for ClickHouse connection 360 | 361 | # 0.8.2 362 | 363 | ### New features 364 | 365 | * Compatibility with Metabase 0.44 366 | -------------------------------------------------------------------------------- /test/metabase/driver/clickhouse_introspection_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.clickhouse-introspection-test 2 | #_{:clj-kondo/ignore [:unsorted-required-namespaces]} 3 | (:require 4 | [clojure.test :refer :all] 5 | [metabase.driver :as driver] 6 | [metabase.driver.common :as driver.common] 7 | [metabase.query-processor :as qp] 8 | [metabase.query-processor.test-util :as qp.test] 9 | [metabase.test :as mt] 10 | [metabase.test.data :as data] 11 | [metabase.test.data.clickhouse :as ctd] 12 | [metabase.test.data.interface :as tx] 13 | [toucan2.tools.with-temp :as t2.with-temp])) 14 | 15 | (use-fixtures :once ctd/create-test-db!) 16 | 17 | (defn- desc-table 18 | [table-name] 19 | (into #{} (map #(select-keys % [:name :database-type :base-type :database-required]) 20 | (:fields (ctd/do-with-test-db 21 | #(driver/describe-table :clickhouse % {:name table-name})))))) 22 | 23 | (deftest ^:parallel clickhouse-base-types-test-enums 24 | (mt/test-driver 25 | :clickhouse 26 | (testing "enums" 27 | (let [table-name "enums_base_types"] 28 | (is (= #{{:base-type :type/Text, 29 | :database-required false, 30 | :database-type "Nullable(Enum8('America/New_York' = 1))", 31 | :name "c1"} 32 | {:base-type :type/Text, 33 | :database-required true, 34 | :database-type "Enum8('BASE TABLE' = 1, 'VIEW' = 2, 'FOREIGN TABLE' = 3, 'LOCAL TEMPORARY' = 4, 'SYSTEM VIEW' = 5)", 35 | :name "c2"} 36 | {:base-type :type/Text, 37 | :database-required true, 38 | :database-type "Enum8('NO' = 1, 'YES' = 2)", 39 | :name "c3"} 40 | {:base-type :type/Text, 41 | :database-required true, 42 | :database-type "Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2)", 43 | :name "c4"} 44 | {:base-type :type/Text, 45 | :database-required false, 46 | :database-type "Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2))", 47 | :name "c5"} 48 | {:base-type :type/Text, 49 | :database-required false, 50 | :database-type "Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2))", 51 | :name "c6"}} 52 | (desc-table table-name))))))) 53 | 54 | (deftest ^:parallel clickhouse-base-types-test-dates 55 | (mt/test-driver 56 | :clickhouse 57 | (testing "dates" 58 | (let [table-name "date_base_types"] 59 | (is (= #{{:base-type :type/Date, 60 | :database-required true, 61 | :database-type "Date", 62 | :name "c1"} 63 | {:base-type :type/Date, 64 | :database-required true, 65 | :database-type "Date32", 66 | :name "c2"} 67 | {:base-type :type/Date, 68 | :database-required false, 69 | :database-type "Nullable(Date)", 70 | :name "c3"} 71 | {:base-type :type/Date, 72 | :database-required false, 73 | :database-type "Nullable(Date32)", 74 | :name "c4"}} 75 | (desc-table table-name))))))) 76 | 77 | (deftest ^:parallel clickhouse-base-types-test-datetimes 78 | (mt/test-driver 79 | :clickhouse 80 | (testing "datetimes" 81 | (let [table-name "datetime_base_types"] 82 | (is (= #{{:base-type :type/DateTimeWithLocalTZ, 83 | :database-required false, 84 | :database-type "Nullable(DateTime('America/New_York'))", 85 | :name "c1"} 86 | {:base-type :type/DateTimeWithLocalTZ, 87 | :database-required true, 88 | :database-type "DateTime('America/New_York')", 89 | :name "c2"} 90 | {:base-type :type/DateTimeWithLocalTZ, 91 | :database-required true, 92 | :database-type "DateTime", 93 | :name "c3"} 94 | {:base-type :type/DateTimeWithLocalTZ, 95 | :database-required true, 96 | :database-type "DateTime64(3)", 97 | :name "c4"} 98 | {:base-type :type/DateTimeWithLocalTZ, 99 | :database-required true, 100 | :database-type "DateTime64(9, 'America/New_York')", 101 | :name "c5"} 102 | {:base-type :type/DateTimeWithLocalTZ, 103 | :database-required false, 104 | :database-type "Nullable(DateTime64(6, 'America/New_York'))", 105 | :name "c6"} 106 | {:base-type :type/DateTimeWithLocalTZ, 107 | :database-required false, 108 | :database-type "Nullable(DateTime64(0))", 109 | :name "c7"} 110 | {:base-type :type/DateTimeWithLocalTZ, 111 | :database-required false, 112 | :database-type "Nullable(DateTime)", 113 | :name "c8"}} 114 | (desc-table table-name))))))) 115 | 116 | (deftest ^:parallel clickhouse-base-types-test-integers 117 | (mt/test-driver 118 | :clickhouse 119 | (testing "integers" 120 | (let [table-name "integer_base_types"] 121 | (is (= #{{:base-type :type/Integer, 122 | :database-required true, 123 | :database-type "UInt8", 124 | :name "c1"} 125 | {:base-type :type/Integer, 126 | :database-required true, 127 | :database-type "UInt16", 128 | :name "c2"} 129 | {:base-type :type/Integer, 130 | :database-required true, 131 | :database-type "UInt32", 132 | :name "c3"} 133 | {:base-type :type/BigInteger, 134 | :database-required true, 135 | :database-type "UInt64", 136 | :name "c4"} 137 | {:base-type :type/*, 138 | :database-required true, 139 | :database-type "UInt128", 140 | :name "c5"} 141 | {:base-type :type/*, 142 | :database-required true, 143 | :database-type "UInt256", 144 | :name "c6"} 145 | {:base-type :type/Integer, 146 | :database-required true, 147 | :database-type "Int8", 148 | :name "c7"} 149 | {:base-type :type/Integer, 150 | :database-required true, 151 | :database-type "Int16", 152 | :name "c8"} 153 | {:base-type :type/Integer, 154 | :database-required true, 155 | :database-type "Int32", 156 | :name "c9"} 157 | {:base-type :type/BigInteger, 158 | :database-required true, 159 | :database-type "Int64", 160 | :name "c10"} 161 | {:base-type :type/*, 162 | :database-required true, 163 | :database-type "Int128", 164 | :name "c11"} 165 | {:base-type :type/*, 166 | :database-required true, 167 | :database-type "Int256", 168 | :name "c12"} 169 | {:base-type :type/Integer, 170 | :database-required false, 171 | :database-type "Nullable(Int32)", 172 | :name "c13"}} 173 | (desc-table table-name))))))) 174 | 175 | (deftest ^:parallel clickhouse-base-types-test-numerics 176 | (mt/test-driver 177 | :clickhouse 178 | (testing "numerics" 179 | (let [table-name "numeric_base_types"] 180 | (is (= #{{:base-type :type/Float, 181 | :database-required true, 182 | :database-type "Float32", 183 | :name "c1"} 184 | {:base-type :type/Float, 185 | :database-required true, 186 | :database-type "Float64", 187 | :name "c2"} 188 | {:base-type :type/Decimal, 189 | :database-required true, 190 | :database-type "Decimal(4, 2)", 191 | :name "c3"} 192 | {:base-type :type/Decimal, 193 | :database-required true, 194 | :database-type "Decimal(9, 7)", 195 | :name "c4"} 196 | {:base-type :type/Decimal, 197 | :database-required true, 198 | :database-type "Decimal(18, 12)", 199 | :name "c5"} 200 | {:base-type :type/Decimal, 201 | :database-required true, 202 | :database-type "Decimal(38, 24)", 203 | :name "c6"} 204 | {:base-type :type/Decimal, 205 | :database-required true, 206 | :database-type "Decimal(76, 42)", 207 | :name "c7"} 208 | {:base-type :type/Float, 209 | :database-required false, 210 | :database-type "Nullable(Float32)", 211 | :name "c8"} 212 | {:base-type :type/Decimal, 213 | :database-required false, 214 | :database-type "Nullable(Decimal(4, 2))", 215 | :name "c9"} 216 | {:base-type :type/Decimal, 217 | :database-required false, 218 | :database-type "Nullable(Decimal(76, 42))", 219 | :name "c10"}} 220 | (desc-table table-name))))))) 221 | 222 | (deftest ^:parallel clickhouse-base-types-test-strings 223 | (mt/test-driver 224 | :clickhouse 225 | (testing "strings" 226 | (let [table-name "string_base_types"] 227 | (is (= #{{:base-type :type/Text, 228 | :database-required true, 229 | :database-type "String", 230 | :name "c1"} 231 | {:base-type :type/Text, 232 | :database-required true, 233 | :database-type "LowCardinality(String)", 234 | :name "c2"} 235 | {:base-type :type/TextLike, 236 | :database-required true, 237 | :database-type "FixedString(32)", 238 | :name "c3"} 239 | {:base-type :type/Text, 240 | :database-required false, 241 | :database-type "Nullable(String)", 242 | :name "c4"} 243 | {:base-type :type/TextLike, 244 | :database-required true, 245 | :database-type "LowCardinality(FixedString(4))", 246 | :name "c5"}} 247 | (desc-table table-name))))))) 248 | 249 | (deftest ^:parallel clickhouse-base-types-test-arrays 250 | (mt/test-driver 251 | :clickhouse 252 | (testing "arrays" 253 | (let [table-name "array_base_types"] 254 | (is (= #{{:base-type :type/Array, 255 | :database-required true, 256 | :database-type "Array(String)", 257 | :name "c1"} 258 | {:base-type :type/Array, 259 | :database-required true, 260 | :database-type "Array(Nullable(Int32))", 261 | :name "c2"} 262 | {:base-type :type/Array, 263 | :database-required true, 264 | :database-type "Array(Array(LowCardinality(FixedString(32))))", 265 | :name "c3"} 266 | {:base-type :type/Array, 267 | :database-required true, 268 | :database-type "Array(Array(Array(String)))", 269 | :name "c4"}} 270 | (desc-table table-name))))))) 271 | 272 | (deftest ^:parallel clickhouse-base-types-test-low-cardinality-nullable 273 | (mt/test-driver 274 | :clickhouse 275 | (testing "low cardinality nullable" 276 | (let [table-name "low_cardinality_nullable_base_types"] 277 | (is (= #{{:base-type :type/Text, 278 | :database-required true, 279 | :database-type "LowCardinality(Nullable(String))", 280 | :name "c1"} 281 | {:base-type :type/TextLike, 282 | :database-required true, 283 | :database-type "LowCardinality(Nullable(FixedString(16)))", 284 | :name "c2"}} 285 | (desc-table table-name))))))) 286 | 287 | (deftest ^:parallel clickhouse-base-types-test-misc 288 | (mt/test-driver 289 | :clickhouse 290 | (testing "everything else" 291 | (let [table-name "misc_base_types"] 292 | (is (= #{{:base-type :type/Boolean, 293 | :database-required true, 294 | :database-type "Bool", 295 | :name "c1"} 296 | {:base-type :type/UUID, 297 | :database-required true, 298 | :database-type "UUID", 299 | :name "c2"} 300 | {:base-type :type/IPAddress, 301 | :database-required true, 302 | :database-type "IPv4", 303 | :name "c3"} 304 | {:base-type :type/IPAddress, 305 | :database-required true, 306 | :database-type "IPv6", 307 | :name "c4"} 308 | {:base-type :type/Dictionary, 309 | :database-required true, 310 | :database-type "Map(Int32, String)", 311 | :name "c5"} 312 | {:base-type :type/Boolean, 313 | :database-required false, 314 | :database-type "Nullable(Bool)", 315 | :name "c6"} 316 | {:base-type :type/UUID, 317 | :database-required false, 318 | :database-type "Nullable(UUID)", 319 | :name "c7"} 320 | {:base-type :type/IPAddress, 321 | :database-required false, 322 | :database-type "Nullable(IPv4)", 323 | :name "c8"} 324 | {:base-type :type/IPAddress, 325 | :database-required false, 326 | :database-type "Nullable(IPv6)", 327 | :name "c9"} 328 | {:base-type :type/*, 329 | :database-required true, 330 | :database-type "Tuple(String, Int32)", 331 | :name "c10"}} 332 | (desc-table table-name))))))) 333 | 334 | (deftest ^:parallel clickhouse-boolean-type-metadata 335 | (mt/test-driver 336 | :clickhouse 337 | (let [result (-> {:query "SELECT false, 123, true"} mt/native-query qp/process-query) 338 | [[c1 _ c3]] (-> result qp.test/rows)] 339 | (testing "column should be of type :type/Boolean" 340 | (is (= :type/Boolean (-> result :data :results_metadata :columns first :base_type))) 341 | (is (= :type/Boolean (transduce identity (driver.common/values->base-type) [c1, c3]))) 342 | (is (= :type/Boolean (driver.common/class->base-type (class c1)))))))) 343 | 344 | (def ^:private base-field 345 | {:database-is-auto-increment false 346 | :json-unfolding false 347 | :database-required true}) 348 | 349 | (deftest ^:parallel clickhouse-filtered-aggregate-functions-test-table-metadata 350 | (mt/test-driver 351 | :clickhouse 352 | (is (= {:name "aggregate_functions_filter_test" 353 | :fields #{(merge base-field 354 | {:name "idx" 355 | :database-type "UInt8" 356 | :base-type :type/Integer 357 | :database-position 0}) 358 | (merge base-field 359 | {:name "lowest_value" 360 | :database-type "SimpleAggregateFunction(min, UInt8)" 361 | :base-type :type/Integer 362 | :database-position 2}) 363 | (merge base-field 364 | {:name "count" 365 | :database-type "SimpleAggregateFunction(sum, Int64)" 366 | :base-type :type/BigInteger 367 | :database-position 3})}} 368 | (ctd/do-with-test-db 369 | (fn [db] 370 | (driver/describe-table :clickhouse db {:name "aggregate_functions_filter_test"}))))))) 371 | 372 | (deftest ^:parallel clickhouse-filtered-aggregate-functions-test-result-set 373 | (mt/test-driver 374 | :clickhouse 375 | (is (= [[42 144 255255]] 376 | (qp.test/formatted-rows 377 | [int int int] 378 | :format-nil-values 379 | (ctd/do-with-test-db 380 | (fn [db] 381 | (data/with-db db 382 | (data/run-mbql-query 383 | aggregate_functions_filter_test 384 | {}))))))))) 385 | 386 | (def ^:private test-tables 387 | #{{:description nil, 388 | :name "table1", 389 | :schema "metabase_db_scan_test"} 390 | {:description nil, 391 | :name "table2", 392 | :schema "metabase_db_scan_test"}}) 393 | 394 | (deftest ^:parallel clickhouse-describe-database-single 395 | (mt/test-driver 396 | :clickhouse 397 | (t2.with-temp/with-temp 398 | [:model/Database db {:engine :clickhouse 399 | :details (merge {:scan-all-databases nil} 400 | (tx/dbdef->connection-details 401 | :clickhouse :db 402 | {:database-name "metabase_db_scan_test"}))}] 403 | (let [describe-result (driver/describe-database :clickhouse db)] 404 | (is (= {:tables test-tables} describe-result)))))) 405 | 406 | (deftest ^:parallel clickhouse-describe-database-all 407 | (mt/test-driver 408 | :clickhouse 409 | (t2.with-temp/with-temp 410 | [:model/Database db {:engine :clickhouse 411 | :details (merge {:scan-all-databases true} 412 | (tx/dbdef->connection-details 413 | :clickhouse :db 414 | {:database-name "default"}))}] 415 | (let [describe-result (driver/describe-database :clickhouse db)] 416 | ;; check the existence of at least some test tables here 417 | (doseq [table test-tables] 418 | (is (contains? (:tables describe-result) table))) 419 | ;; should not contain any ClickHouse system tables 420 | (is (not (some #(= (:schema %) "system") 421 | (:tables describe-result)))) 422 | (is (not (some #(= (:schema %) "information_schema") 423 | (:tables describe-result)))) 424 | (is (not (some #(= (:schema %) "INFORMATION_SCHEMA") 425 | (:tables describe-result)))))))) 426 | 427 | (deftest ^:parallel clickhouse-describe-database-multiple 428 | (mt/test-driver 429 | :clickhouse 430 | (t2.with-temp/with-temp 431 | [:model/Database db {:engine :clickhouse 432 | :details (tx/dbdef->connection-details 433 | :clickhouse :db 434 | {:database-name "metabase_db_scan_test information_schema"})}] 435 | (let [{:keys [tables] :as _describe-result} 436 | (driver/describe-database :clickhouse db) 437 | tables-table {:name "tables" 438 | :description nil 439 | :schema "information_schema"} 440 | columns-table {:name "columns" 441 | :description nil 442 | :schema "information_schema"}] 443 | 444 | ;; tables from `metabase_db_scan_test` 445 | (doseq [table test-tables] 446 | (is (contains? tables table))) 447 | 448 | ;; tables from `information_schema` 449 | (is (contains? tables tables-table)) 450 | (is (contains? tables columns-table)))))) 451 | -------------------------------------------------------------------------------- /src/metabase/driver/clickhouse_qp.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.clickhouse-qp 2 | "CLickHouse driver: QueryProcessor-related definition" 3 | #_{:clj-kondo/ignore [:unsorted-required-namespaces]} 4 | (:require [clojure.string :as str] 5 | [honey.sql :as sql] 6 | [java-time.api :as t] 7 | [metabase.driver.clickhouse-nippy] 8 | [metabase.driver.clickhouse-version :as clickhouse-version] 9 | [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] 10 | [metabase.driver.sql.query-processor :as sql.qp :refer [add-interval-honeysql-form]] 11 | [metabase.driver.sql.util :as sql.u] 12 | [metabase.driver.sql.util.unprepare :as unprepare] 13 | [metabase.legacy-mbql.util :as mbql.u] 14 | [metabase.query-processor.timezone :as qp.timezone] 15 | [metabase.util :as u] 16 | [metabase.util.date-2 :as u.date] 17 | [metabase.util.honey-sql-2 :as h2x]) 18 | (:import [java.sql ResultSet ResultSetMetaData Types] 19 | [java.time 20 | LocalDate 21 | LocalDateTime 22 | LocalTime 23 | OffsetDateTime 24 | OffsetTime 25 | ZonedDateTime] 26 | java.util.Arrays)) 27 | 28 | ;; (set! *warn-on-reflection* true) ;; isn't enabled because of Arrays/toString call 29 | 30 | (defmethod sql.qp/quote-style :clickhouse [_] :mysql) 31 | 32 | ;; without try, there might be test failures when QP is not yet initialized 33 | ;; e.g., when a test is preparing the dataset 34 | (defn- get-report-timezone-id-safely 35 | [] 36 | (try 37 | (qp.timezone/report-timezone-id-if-supported) 38 | (catch Throwable _e nil))) 39 | 40 | ;; datetime('europe/amsterdam') -> europe/amsterdam 41 | (defn- extract-datetime-timezone 42 | [db-type] 43 | (when (and db-type (string? db-type)) 44 | (cond 45 | ;; e.g. DateTime64(3, 'Europe/Amsterdam') 46 | (str/starts-with? db-type "datetime64") 47 | (if (> (count db-type) 17) (subs db-type 15 (- (count db-type) 2)) nil) 48 | ;; e.g. DateTime('Europe/Amsterdam') 49 | (str/starts-with? db-type "datetime") 50 | (if (> (count db-type) 12) (subs db-type 10 (- (count db-type) 2)) nil) 51 | ;; _ 52 | :else nil))) 53 | 54 | (defn- remove-low-cardinality-and-nullable 55 | [db-type] 56 | (when (and db-type (string? db-type)) 57 | (let [db-type-lowercase (u/lower-case-en db-type) 58 | without-low-car (if (str/starts-with? db-type-lowercase "lowcardinality(") 59 | (subs db-type-lowercase 15 (- (count db-type-lowercase) 1)) 60 | db-type-lowercase) 61 | without-nullable (if (str/starts-with? without-low-car "nullable(") 62 | (subs without-low-car 9 (- (count without-low-car) 1)) 63 | without-low-car)] 64 | without-nullable))) 65 | 66 | (defn- in-report-timezone 67 | [expr] 68 | (let [report-timezone (get-report-timezone-id-safely) 69 | lower (u/lower-case-en (h2x/database-type expr)) 70 | db-type (remove-low-cardinality-and-nullable lower)] 71 | (if (and report-timezone (string? db-type) (str/starts-with? db-type "datetime")) 72 | (let [timezone (extract-datetime-timezone db-type)] 73 | (if (not (= timezone (u/lower-case-en report-timezone))) 74 | [:'toTimeZone expr (h2x/literal report-timezone)] 75 | expr)) 76 | expr))) 77 | 78 | (defmethod sql.qp/date [:clickhouse :default] 79 | [_ _ expr] 80 | expr) 81 | 82 | ;;; ------------------------------------------------------------------------------------ 83 | ;;; Extract functions 84 | ;;; ------------------------------------------------------------------------------------ 85 | 86 | (defn- date-extract 87 | [ch-fn expr db-type] 88 | (-> [ch-fn (in-report-timezone expr)] 89 | (h2x/with-database-type-info db-type))) 90 | 91 | (defmethod sql.qp/date [:clickhouse :day-of-week] 92 | [_ _ expr] 93 | ;; a tick in the function name prevents HSQL2 to make the function call UPPERCASE 94 | ;; https://cljdoc.org/d/com.github.seancorfield/honeysql/2.4.1011/doc/getting-started/other-databases#clickhouse 95 | (sql.qp/adjust-day-of-week 96 | :clickhouse (date-extract :'toDayOfWeek expr "uint8"))) 97 | 98 | (defmethod sql.qp/date [:clickhouse :month-of-year] 99 | [_ _ expr] 100 | (date-extract :'toMonth expr "uint8")) 101 | 102 | (defmethod sql.qp/date [:clickhouse :minute-of-hour] 103 | [_ _ expr] 104 | (date-extract :'toMinute expr "uint8")) 105 | 106 | (defmethod sql.qp/date [:clickhouse :hour-of-day] 107 | [_ _ expr] 108 | (date-extract :'toHour expr "uint8")) 109 | 110 | (defmethod sql.qp/date [:clickhouse :day-of-month] 111 | [_ _ expr] 112 | (date-extract :'toDayOfMonth expr "uint8")) 113 | 114 | (defmethod sql.qp/date [:clickhouse :day-of-year] 115 | [_ _ expr] 116 | (date-extract :'toDayOfYear expr "uint16")) 117 | 118 | (defmethod sql.qp/date [:clickhouse :week-of-year-iso] 119 | [_ _ expr] 120 | (date-extract :'toISOWeek expr "uint8")) 121 | 122 | (defmethod sql.qp/date [:clickhouse :quarter-of-year] 123 | [_ _ expr] 124 | (date-extract :'toQuarter expr "uint8")) 125 | 126 | (defmethod sql.qp/date [:clickhouse :year-of-era] 127 | [_ _ expr] 128 | (date-extract :'toYear expr "uint16")) 129 | 130 | ;;; ------------------------------------------------------------------------------------ 131 | ;;; Truncate functions 132 | ;;; ------------------------------------------------------------------------------------ 133 | 134 | (defn- date-trunc 135 | [ch-fn expr] 136 | [ch-fn (in-report-timezone expr)]) 137 | 138 | (defn- to-start-of-week 139 | [expr] 140 | (date-trunc :'toMonday expr)) 141 | 142 | (defmethod sql.qp/date [:clickhouse :minute] 143 | [_ _ expr] 144 | (date-trunc :'toStartOfMinute expr)) 145 | 146 | (defmethod sql.qp/date [:clickhouse :hour] 147 | [_ _ expr] 148 | (date-trunc :'toStartOfHour expr)) 149 | 150 | (defmethod sql.qp/date [:clickhouse :day] 151 | [_ _ expr] 152 | (date-trunc :'toStartOfDay expr)) 153 | 154 | (defmethod sql.qp/date [:clickhouse :week] 155 | [driver _ expr] 156 | (sql.qp/adjust-start-of-week driver to-start-of-week expr)) 157 | 158 | (defmethod sql.qp/date [:clickhouse :month] 159 | [_ _ expr] 160 | (date-trunc :'toStartOfMonth expr)) 161 | 162 | (defmethod sql.qp/date [:clickhouse :quarter] 163 | [_ _ expr] 164 | (date-trunc :'toStartOfQuarter expr)) 165 | 166 | (defmethod sql.qp/date [:clickhouse :year] 167 | [_ _ expr] 168 | (date-trunc :'toStartOfYear expr)) 169 | 170 | ;;; ------------------------------------------------------------------------------------ 171 | ;;; Unix timestamps functions 172 | ;;; ------------------------------------------------------------------------------------ 173 | 174 | (defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :seconds] 175 | [_ _ expr] 176 | (h2x/->datetime expr)) 177 | 178 | (defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :milliseconds] 179 | [_ _ expr] 180 | (let [report-timezone (get-report-timezone-id-safely) 181 | inner-expr (h2x// expr 1000)] 182 | (if report-timezone 183 | [:'toDateTime64 inner-expr 3 report-timezone] 184 | [:'toDateTime64 inner-expr 3]))) 185 | 186 | (defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :microseconds] 187 | [_ _ expr] 188 | (let [report-timezone (get-report-timezone-id-safely) 189 | inner-expr [:'toInt64 (h2x// expr 1000)]] 190 | (if report-timezone 191 | [:'fromUnixTimestamp64Milli inner-expr report-timezone] 192 | [:'fromUnixTimestamp64Milli inner-expr]))) 193 | 194 | ;;; ------------------------------------------------------------------------------------ 195 | ;;; HoneySQL forms 196 | ;;; ------------------------------------------------------------------------------------ 197 | 198 | (defmethod sql.qp/->honeysql [:clickhouse :convert-timezone] 199 | [driver [_ arg target-timezone source-timezone]] 200 | (let [expr (sql.qp/->honeysql driver (cond-> arg (string? arg) u.date/parse)) 201 | with-tz-info? (h2x/is-of-type? expr #"(?:nullable\(|lowcardinality\()?(datetime64\(\d, {0,1}'.*|datetime\(.*)") 202 | _ (sql.u/validate-convert-timezone-args with-tz-info? target-timezone source-timezone)] 203 | (if (not with-tz-info?) 204 | [:'plus 205 | expr 206 | [:'toIntervalSecond 207 | [:'minus 208 | [:'timeZoneOffset [:'toTimeZone expr target-timezone]] 209 | [:'timeZoneOffset [:'toTimeZone expr source-timezone]]]]] 210 | [:'toTimeZone expr target-timezone]))) 211 | 212 | (defmethod sql.qp/current-datetime-honeysql-form :clickhouse 213 | [_] 214 | (let [report-timezone (get-report-timezone-id-safely) 215 | [expr db-type] (if report-timezone 216 | [[:'now64 [:raw 9] (h2x/literal report-timezone)] (format "DateTime64(9, '%s')" report-timezone)] 217 | [[:'now64 [:raw 9]] "DateTime64(9)"])] 218 | (h2x/with-database-type-info expr db-type))) 219 | 220 | (defn- date-time-parse-fn 221 | [nano] 222 | (if (zero? nano) :'parseDateTimeBestEffort :'parseDateTime64BestEffort)) 223 | 224 | (defmethod sql.qp/->honeysql [:clickhouse LocalDateTime] 225 | [_ ^java.time.LocalDateTime t] 226 | (let [formatted (t/format "yyyy-MM-dd HH:mm:ss.SSS" t) 227 | report-tz (or (get-report-timezone-id-safely) "UTC")] 228 | (if (zero? (.getNano t)) 229 | [:'parseDateTimeBestEffort formatted report-tz] 230 | [:'parseDateTime64BestEffort formatted 3 report-tz]))) 231 | 232 | (defmethod sql.qp/->honeysql [:clickhouse ZonedDateTime] 233 | [_ ^java.time.ZonedDateTime t] 234 | (let [formatted (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t) 235 | fn (date-time-parse-fn (.getNano t))] 236 | [fn formatted])) 237 | 238 | (defmethod sql.qp/->honeysql [:clickhouse OffsetDateTime] 239 | [_ ^java.time.OffsetDateTime t] 240 | ;; copy-paste due to reflection warnings 241 | (let [formatted (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t) 242 | fn (date-time-parse-fn (.getNano t))] 243 | [fn formatted])) 244 | 245 | (defmethod sql.qp/->honeysql [:clickhouse LocalDate] 246 | [_ ^java.time.LocalDate t] 247 | [:'parseDateTimeBestEffort t]) 248 | 249 | (defn- local-date-time 250 | [^java.time.LocalTime t] 251 | (t/local-date-time (t/local-date 1970 1 1) t)) 252 | 253 | (defmethod sql.qp/->honeysql [:clickhouse LocalTime] 254 | [driver ^java.time.LocalTime t] 255 | (sql.qp/->honeysql driver (local-date-time t))) 256 | 257 | (defmethod sql.qp/->honeysql [:clickhouse OffsetTime] 258 | [driver ^java.time.OffsetTime t] 259 | (sql.qp/->honeysql driver (t/offset-date-time 260 | (local-date-time (.toLocalTime t)) 261 | (.getOffset t)))) 262 | 263 | (defn- args->float64 264 | [args] 265 | (map (fn [arg] [:'toFloat64 (sql.qp/->honeysql :clickhouse arg)]) args)) 266 | 267 | (defn- interval? [expr] 268 | (mbql.u/is-clause? :interval expr)) 269 | 270 | (defmethod sql.qp/->honeysql [:clickhouse :+] 271 | [driver [_ & args]] 272 | (if (some interval? args) 273 | (if-let [[field intervals] (u/pick-first (complement interval?) args)] 274 | (reduce (fn [hsql-form [_ amount unit]] 275 | (add-interval-honeysql-form driver hsql-form amount unit)) 276 | (sql.qp/->honeysql driver field) 277 | intervals) 278 | (throw (ex-info "Summing intervals is not supported" {:args args}))) 279 | (into [:+] (args->float64 args)))) 280 | 281 | (defmethod sql.qp/->honeysql [:clickhouse :log] 282 | [driver [_ field]] 283 | [:'log10 (sql.qp/->honeysql driver field)]) 284 | 285 | (defn- format-expr 286 | [expr] 287 | (first (sql/format-expr (sql.qp/->honeysql :clickhouse expr) {:nested true}))) 288 | 289 | (defmethod sql.qp/->honeysql [:clickhouse :percentile] 290 | [_ [_ field p]] 291 | [:raw (format "quantile(%s)(%s)" (format-expr p) (format-expr field))]) 292 | 293 | (defmethod sql.qp/->honeysql [:clickhouse :regex-match-first] 294 | [driver [_ arg pattern]] 295 | [:'extract (sql.qp/->honeysql driver arg) pattern]) 296 | 297 | (defmethod sql.qp/->honeysql [:clickhouse :stddev] 298 | [driver [_ field]] 299 | [:'stddevPop (sql.qp/->honeysql driver field)]) 300 | 301 | (defmethod sql.qp/->honeysql [:clickhouse :median] 302 | [driver [_ field]] 303 | [:'median (sql.qp/->honeysql driver field)]) 304 | 305 | ;; Substring does not work for Enums, so we need to cast to String 306 | (defmethod sql.qp/->honeysql [:clickhouse :substring] 307 | [driver [_ arg start length]] 308 | (let [str [:'toString (sql.qp/->honeysql driver arg)]] 309 | (if length 310 | [:'substring str 311 | (sql.qp/->honeysql driver start) 312 | (sql.qp/->honeysql driver length)] 313 | [:'substring str 314 | (sql.qp/->honeysql driver start)]))) 315 | 316 | (defmethod sql.qp/->honeysql [:clickhouse :var] 317 | [driver [_ field]] 318 | [:'varPop (sql.qp/->honeysql driver field)]) 319 | 320 | (defmethod sql.qp/->float :clickhouse 321 | [_ value] 322 | [:'toFloat64 value]) 323 | 324 | (defmethod sql.qp/->honeysql [:clickhouse :value] 325 | [driver value] 326 | (let [[_ value {base-type :base_type}] value] 327 | (when (some? value) 328 | (condp #(isa? %2 %1) base-type 329 | :type/IPAddress [:'toIPv4 value] 330 | (sql.qp/->honeysql driver value))))) 331 | 332 | (defmethod sql.qp/->honeysql [:clickhouse :=] 333 | [driver [op field value]] 334 | (let [[qual valuevalue fieldinfo] value 335 | hsql-field (sql.qp/->honeysql driver field) 336 | hsql-value (sql.qp/->honeysql driver value)] 337 | (if (and (isa? qual :value) 338 | (isa? (:base_type fieldinfo) :type/Text) 339 | (nil? valuevalue)) 340 | [:or 341 | [:= hsql-field hsql-value] 342 | [:= [:'empty hsql-field] 1]] 343 | ((get-method sql.qp/->honeysql [:sql :=]) driver [op field value])))) 344 | 345 | (defmethod sql.qp/->honeysql [:clickhouse :!=] 346 | [driver [op field value]] 347 | (let [[qual valuevalue fieldinfo] value 348 | hsql-field (sql.qp/->honeysql driver field) 349 | hsql-value (sql.qp/->honeysql driver value)] 350 | (if (and (isa? qual :value) 351 | (isa? (:base_type fieldinfo) :type/Text) 352 | (nil? valuevalue)) 353 | [:and 354 | [:!= hsql-field hsql-value] 355 | [:= [:'notEmpty hsql-field] 1]] 356 | ((get-method sql.qp/->honeysql [:sql :!=]) driver [op field value])))) 357 | 358 | ;; I do not know why the tests expect nil counts for empty results 359 | ;; but that's how it is :-) 360 | ;; 361 | ;; It would even be better if we could use countIf and sumIf directly 362 | ;; 363 | ;; metabase.query-processor-test.count-where-test 364 | ;; metabase.query-processor-test.share-test 365 | (defmethod sql.qp/->honeysql [:clickhouse :count-where] 366 | [driver [_ pred]] 367 | [:case 368 | [:> [:'count] 0] 369 | [:sum [:case (sql.qp/->honeysql driver pred) 1 :else 0]] 370 | :else nil]) 371 | 372 | (defmethod sql.qp/->honeysql [:clickhouse :sum-where] 373 | [driver [_ field pred]] 374 | [:sum [:case (sql.qp/->honeysql driver pred) (sql.qp/->honeysql driver field) 375 | :else 0]]) 376 | 377 | (defmethod sql.qp/add-interval-honeysql-form :clickhouse 378 | [_ dt amount unit] 379 | (h2x/+ dt [:raw (format "INTERVAL %d %s" (int amount) (name unit))])) 380 | 381 | (defn- clickhouse-string-fn 382 | [fn-name field value options] 383 | (let [hsql-field (sql.qp/->honeysql :clickhouse field) 384 | hsql-value (sql.qp/->honeysql :clickhouse value)] 385 | (if (get options :case-sensitive true) 386 | [fn-name hsql-field hsql-value] 387 | [fn-name [:'lowerUTF8 hsql-field] [:'lowerUTF8 hsql-value]]))) 388 | 389 | (defmethod sql.qp/->honeysql [:clickhouse :starts-with] 390 | [_ [_ field value options]] 391 | (let [starts-with (clickhouse-version/with-min 23 8 392 | (constantly :'startsWithUTF8) 393 | (constantly :'startsWith))] 394 | (clickhouse-string-fn starts-with field value options))) 395 | 396 | (defmethod sql.qp/->honeysql [:clickhouse :ends-with] 397 | [_ [_ field value options]] 398 | (let [ends-with (clickhouse-version/with-min 23 8 399 | (constantly :'endsWithUTF8) 400 | (constantly :'endsWith))] 401 | (clickhouse-string-fn ends-with field value options))) 402 | 403 | (defmethod sql.qp/->honeysql [:clickhouse :contains] 404 | [_ [_ field value options]] 405 | (let [hsql-field (sql.qp/->honeysql :clickhouse field) 406 | hsql-value (sql.qp/->honeysql :clickhouse value) 407 | position-fn (if (get options :case-sensitive true) 408 | :'positionUTF8 409 | :'positionCaseInsensitiveUTF8)] 410 | [:> [position-fn hsql-field hsql-value] 0])) 411 | 412 | (defmethod sql.qp/->honeysql [:clickhouse :datetime-diff] 413 | [driver [_ x y unit]] 414 | (let [x (sql.qp/->honeysql driver x) 415 | y (sql.qp/->honeysql driver y)] 416 | (case unit 417 | ;; Week: Metabase tests expect a bit different result from what `age` provides 418 | (:week) 419 | [:'intDiv [:'dateDiff (h2x/literal :day) (date-trunc :'toStartOfDay x) (date-trunc :'toStartOfDay y)] [:raw 7]] 420 | ;; ------------------------- 421 | (:year :month :quarter :day) 422 | [:'age (h2x/literal unit) (date-trunc :'toStartOfDay x) (date-trunc :'toStartOfDay y)] 423 | ;; ------------------------- 424 | (:hour :minute :second) 425 | [:'age (h2x/literal unit) (in-report-timezone x) (in-report-timezone y)]))) 426 | 427 | ;; We do not have Time data types, so we cheat a little bit 428 | (defmethod sql.qp/cast-temporal-string [:clickhouse :Coercion/ISO8601->Time] 429 | [_driver _special_type expr] 430 | [:'parseDateTimeBestEffort [:'concat "1970-01-01T" expr]]) 431 | 432 | (defmethod sql.qp/cast-temporal-byte [:clickhouse :Coercion/ISO8601->Time] 433 | [_driver _special_type expr] 434 | expr) 435 | 436 | ;;; ------------------------------------------------------------------------------------ 437 | ;;; JDBC-related functions 438 | ;;; ------------------------------------------------------------------------------------ 439 | 440 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TINYINT] 441 | [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] 442 | (fn [] 443 | (.getObject rs i))) 444 | 445 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/SMALLINT] 446 | [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] 447 | (fn [] 448 | (.getObject rs i))) 449 | 450 | ;; This is for tests only - some of them expect nil values 451 | ;; getInt/getLong return 0 in case of a NULL value in the result set 452 | ;; the only way to check if it was actually NULL - call ResultSet.wasNull afterwards 453 | (defn- with-null-check 454 | [^ResultSet rs value] 455 | (if (.wasNull rs) nil value)) 456 | 457 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/BIGINT] 458 | [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] 459 | (fn [] 460 | (with-null-check rs (.getBigDecimal rs i)))) 461 | 462 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/INTEGER] 463 | [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] 464 | (fn [] 465 | (with-null-check rs (.getLong rs i)))) 466 | 467 | (def ^:private utc-zone-id (java.time.ZoneId/of "UTC")) 468 | (defn- zdt-in-report-timezone 469 | [^ZonedDateTime zdt] 470 | (let [maybe-report-timezone (get-report-timezone-id-safely)] 471 | (if maybe-report-timezone 472 | (.withZoneSameInstant zdt (java.time.ZoneId/of maybe-report-timezone)) 473 | (if (= (.getId (.getZone zdt)) "GMT0") ;; for test purposes only; GMT0 is a legacy tz 474 | (.withZoneSameInstant zdt utc-zone-id) 475 | zdt)))) 476 | 477 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/DATE] 478 | [_ ^ResultSet rs ^ResultSetMetaData _rsmeta ^Integer i] 479 | (fn [] 480 | (when-let [sql-date (.getDate rs i)] 481 | (.toLocalDate sql-date)))) 482 | 483 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIMESTAMP] 484 | [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] 485 | (fn [] 486 | (when-let [zdt (.getObject rs i ZonedDateTime)] 487 | (let [db-type (remove-low-cardinality-and-nullable (.getColumnTypeName rsmeta i))] 488 | (if (= db-type "datetime64(3, 'gmt0')") 489 | ;; a hack for some MB test assertions only; GMT0 is a legacy tz 490 | (.toLocalDateTime (zdt-in-report-timezone zdt)) 491 | ;; this is the normal behavior 492 | (.toOffsetDateTime (.withZoneSameInstant 493 | (zdt-in-report-timezone zdt) 494 | utc-zone-id))))))) 495 | 496 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIME] 497 | [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] 498 | (fn [] 499 | (.getObject rs i OffsetTime))) 500 | 501 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/NUMERIC] 502 | [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] 503 | (fn [] 504 | ; count is NUMERIC cause UInt64 is too large for the canonical SQL BIGINT, 505 | ; and defaults to BigDecimal, but we want it to be coerced to java Long 506 | ; cause it still fits and the tests are expecting that 507 | (if (= (.getColumnLabel rsmeta i) "count") 508 | (.getLong rs i) 509 | (.getBigDecimal rs i)))) 510 | 511 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/ARRAY] 512 | [_ ^ResultSet rs ^ResultSetMetaData _rsmeta ^Integer i] 513 | (fn [] 514 | (when-let [arr (.getArray rs i)] 515 | (Arrays/deepToString (.getArray arr))))) 516 | 517 | (defn- ipv4-column->string 518 | [^ResultSet rs ^Integer i] 519 | (when-let [inet-address (.getObject rs i java.net.Inet4Address)] 520 | (.getHostAddress inet-address))) 521 | 522 | (defn- ipv6-column->string 523 | [^ResultSet rs ^Integer i] 524 | (when-let [inet-address (.getObject rs i java.net.Inet6Address)] 525 | (.getHostAddress inet-address))) 526 | 527 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/OTHER] 528 | [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] 529 | (fn [] 530 | (let [normalized-db-type (remove-low-cardinality-and-nullable 531 | (.getColumnTypeName rsmeta i))] 532 | (cond 533 | (= normalized-db-type "ipv4") 534 | (ipv4-column->string rs i) 535 | (= normalized-db-type "ipv6") 536 | (ipv6-column->string rs i) 537 | ;; _ 538 | :else (.getObject rs i))))) 539 | 540 | (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/VARCHAR] 541 | [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] 542 | (fn [] 543 | (let [normalized-db-type (remove-low-cardinality-and-nullable 544 | (.getColumnTypeName rsmeta i))] 545 | (cond 546 | ;; Enum8/Enum16 547 | (str/starts-with? normalized-db-type "enum") 548 | (.getString rs i) 549 | ;; _ 550 | :else (.getObject rs i))))) 551 | 552 | (defmethod unprepare/unprepare-value [:clickhouse LocalDate] 553 | [_ t] 554 | (format "'%s'" (t/format "yyyy-MM-dd" t))) 555 | 556 | (defmethod unprepare/unprepare-value [:clickhouse LocalTime] 557 | [_ t] 558 | (format "'%s'" (t/format "HH:mm:ss.SSS" t))) 559 | 560 | (defmethod unprepare/unprepare-value [:clickhouse OffsetTime] 561 | [_ t] 562 | (format "'%s'" (t/format "HH:mm:ss.SSSZZZZZ" t))) 563 | 564 | (defmethod unprepare/unprepare-value [:clickhouse LocalDateTime] 565 | [_ t] 566 | (format "'%s'" (t/format "yyyy-MM-dd HH:mm:ss.SSS" t))) 567 | 568 | (defmethod unprepare/unprepare-value [:clickhouse OffsetDateTime] 569 | [_ ^OffsetDateTime t] 570 | (format "%s('%s')" 571 | (if (zero? (.getNano t)) "parseDateTimeBestEffort" "parseDateTime64BestEffort") 572 | (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) 573 | 574 | (defmethod unprepare/unprepare-value [:clickhouse ZonedDateTime] 575 | [_ t] 576 | (format "'%s'" (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) 577 | --------------------------------------------------------------------------------