├── .github └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── NOTICE ├── README.md ├── all-in-one.Dockerfile ├── docker ├── README.md ├── config │ ├── datasource.jschema │ ├── datasources │ │ ├── datasource.json.example │ │ └── script.json │ ├── httpd.json │ ├── queries │ │ └── query.json.example │ ├── query.jschema │ ├── schema.jschema │ ├── schemas │ │ └── schema.json.example │ ├── server.json │ └── vertx.json ├── docker-entrypoint.sh ├── logging.properties ├── scripts │ ├── one-two-three.js │ └── show-query-logs.sql └── set-env.sh.example ├── misc ├── license-header.template ├── license-mappings.xml ├── notice.template ├── perf-test │ ├── clickhouse │ │ ├── config │ │ │ └── perf_test_config.xml │ │ └── init.sql │ ├── docker-compose.yml │ ├── jdbc-bridge │ │ ├── config │ │ │ ├── datasources │ │ │ │ └── mariadb.json │ │ │ ├── queries │ │ │ │ ├── constant.json │ │ │ │ └── small-table.json │ │ │ └── schemas │ │ │ │ ├── simple-num.json │ │ │ │ └── simple-row.json │ │ └── scripts │ │ │ ├── constant.sql │ │ │ └── small-table.sql │ ├── mariadb │ │ └── init.sql │ └── results │ │ ├── clickhouse(patched)_10k-rows-query(jdbc).txt │ │ ├── clickhouse(patched)_10k-rows-query(jdbc-dual).txt │ │ ├── clickhouse(patched)_constant-query(jdbc).txt │ │ ├── clickhouse(patched)_constant-query(jdbc-dual).txt │ │ ├── clickhouse_10k-rows-query(jdbc).txt │ │ ├── clickhouse_10k-rows-query(mysql).txt │ │ ├── clickhouse_10k-rows-query(remote).txt │ │ ├── clickhouse_10k-rows-query(url).txt │ │ ├── clickhouse_10k-rows-query.txt │ │ ├── clickhouse_constant-query(jdbc).txt │ │ ├── clickhouse_constant-query(mysql).txt │ │ ├── clickhouse_constant-query(remote).txt │ │ ├── clickhouse_constant-query(url).txt │ │ ├── clickhouse_constant-query.txt │ │ ├── clickhouse_ping.txt │ │ ├── clickhouse_url(clickhouse).txt │ │ ├── clickhouse_url(jdbc-bridge).txt │ │ └── jdbc-bridge_ping.txt └── quick-start │ ├── ch-server │ └── config │ │ └── jdbc_bridge_config.xml │ ├── docker-compose.yml │ └── jdbc-bridge │ ├── config │ ├── datasources │ │ ├── ch-server.json │ │ ├── elasticsearch.json │ │ ├── mariadb10.json │ │ ├── mysql5.json │ │ ├── mysql8.json │ │ └── postgres13.json │ ├── queries │ │ └── show-query-logs.json │ └── schemas │ │ └── query-log.json │ └── scripts │ └── show-query-logs.sql ├── pom.xml └── src ├── changelog ├── main ├── bin │ └── clickhouse-jdbc-bridge ├── java │ └── com │ │ └── clickhouse │ │ └── jdbcbridge │ │ ├── JdbcBridgeVerticle.java │ │ ├── core │ │ ├── BaseRepository.java │ │ ├── ByteBuffer.java │ │ ├── ColumnDefinition.java │ │ ├── DataAccessException.java │ │ ├── DataSourceStats.java │ │ ├── DataTableReader.java │ │ ├── DataType.java │ │ ├── DataTypeConverter.java │ │ ├── DataTypeMapping.java │ │ ├── DefaultValues.java │ │ ├── DnsResolver.java │ │ ├── ExpandedUrlClassLoader.java │ │ ├── Extension.java │ │ ├── ExtensionManager.java │ │ ├── ManagedEntity.java │ │ ├── NamedDataSource.java │ │ ├── NamedQuery.java │ │ ├── NamedSchema.java │ │ ├── QueryParameters.java │ │ ├── QueryParser.java │ │ ├── Reloadable.java │ │ ├── Repository.java │ │ ├── RepositoryManager.java │ │ ├── ResponseWriter.java │ │ ├── StreamOptions.java │ │ ├── TableDefinition.java │ │ ├── TypedParameter.java │ │ ├── UsageStats.java │ │ └── Utils.java │ │ └── impl │ │ ├── ConfigDataSource.java │ │ ├── DefaultDataTypeConverter.java │ │ ├── DefaultRepositoryManager.java │ │ ├── JdbcDataSource.java │ │ ├── JsonFileRepository.java │ │ └── ScriptDataSource.java └── resources │ ├── META-INF │ └── services │ │ ├── com.clickhouse.jdbcbridge.core.DataTypeConverter │ │ └── com.clickhouse.jdbcbridge.core.RepositoryManager │ └── logging.properties └── test ├── java └── com │ └── clickhouse │ └── jdbcbridge │ ├── JdbcBridgeVerticleTest.java │ ├── core │ ├── ByteBufferTest.java │ ├── ColumnDefinitionTest.java │ ├── DataTypeMappingTest.java │ ├── DnsResolverTest.java │ ├── ExpandedUrlClassLoaderTest.java │ ├── ExtensionTest.java │ ├── NamedDataSourceTest.java │ ├── QueryParametersTest.java │ ├── QueryParserTest.java │ ├── TableDefinitionTest.java │ ├── TypedParameterTest.java │ └── UtilsTest.java │ └── impl │ ├── JsonFileRepositoryTest.java │ └── ScriptDataSourceTest.java └── resources ├── datasources ├── test-datasource.json ├── test-legacy.json └── test-nds.json ├── logging.properties ├── server.json ├── simple.query ├── sit ├── ch-server │ └── jdbc-bridge.xml └── jdbc-bridge │ └── config │ └── datasources │ ├── mariadb.json │ └── postgresql.json └── test.json /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - develop 8 | paths-ignore: 9 | - "**.md" 10 | - "misc/**" 11 | 12 | pull_request: 13 | types: 14 | - opened 15 | - synchronize 16 | - reopened 17 | paths-ignore: 18 | - "**.md" 19 | - "misc/**" 20 | 21 | jobs: 22 | java: 23 | name: "Build and verify Java code..." 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v2 27 | - name: Set up JDK 1.8 28 | uses: actions/setup-java@v1 29 | with: 30 | java-version: 1.8 31 | - name: Build with Maven 32 | run: mvn --batch-mode --update-snapshots verify 33 | 34 | docker: 35 | name: "Build and verify Dockerfile..." 36 | runs-on: ubuntu-latest 37 | steps: 38 | - name: Checkout 39 | uses: actions/checkout@v2 40 | - name: Set up QEMU 41 | uses: docker/setup-qemu-action@v1 42 | - name: Set up Docker Buildx 43 | uses: docker/setup-buildx-action@v1 44 | - name: Build Default Docker Image 45 | uses: docker/build-push-action@v2 46 | with: 47 | context: . 48 | file: ./Dockerfile 49 | push: false 50 | tags: my/clickhouse-jdbc-bridge:latest 51 | - name: Build All-in-one Docker Image 52 | uses: docker/build-push-action@v2 53 | with: 54 | context: . 55 | file: ./all-in-one.Dockerfile 56 | push: false 57 | tags: my/clickhouse-jdbc-bridge:latest 58 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | major: 7 | description: "Major version" 8 | required: true 9 | default: "2" 10 | minor: 11 | description: "Minor version" 12 | required: false 13 | default: "1" 14 | patch: 15 | description: "Patch" 16 | required: false 17 | default: "0" 18 | organization: 19 | description: "DockerHub organization" 20 | required: false 21 | default: "clickhouse" 22 | 23 | jobs: 24 | pre-release: 25 | name: "Pre Release" 26 | runs-on: "ubuntu-latest" 27 | 28 | steps: 29 | - name: Check out Git repository 30 | uses: actions/checkout@v2 31 | - name: Install Java and Maven 32 | uses: actions/setup-java@v1 33 | with: 34 | java-version: 1.8 35 | - run: sed -i -e 's|^\( \).*\(\)$|\1${{ github.event.inputs.major }}.${{ github.event.inputs.minor }}.${{ github.event.inputs.patch }}\2|' pom.xml 36 | - name: Release Maven package 37 | uses: samuelmeuli/action-maven-publish@v1 38 | with: 39 | maven_profiles: release 40 | maven_args: -Drevision=${{ github.event.inputs.major }}.${{ github.event.inputs.minor }}.${{ github.event.inputs.patch }} --batch-mode 41 | gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} 42 | gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} 43 | nexus_username: ${{ secrets.SONATYPE_USER }} 44 | nexus_password: ${{ secrets.SONATYPE_PASSWD }} 45 | - name: Create Pre-release on Github 46 | uses: "zhicwu/action-automatic-releases@latest" 47 | with: 48 | repo_token: "${{ secrets.GITHUB_TOKEN }}" 49 | automatic_release_tag: "v${{ github.event.inputs.major }}.${{ github.event.inputs.minor }}.${{ github.event.inputs.patch }}" 50 | prerelease: true 51 | title: "Release v${{ github.event.inputs.major }}.${{ github.event.inputs.minor }}.${{ github.event.inputs.patch }}" 52 | files: | 53 | LICENSE 54 | NOTICE 55 | target/clickhouse*.jar 56 | target/*.deb 57 | target/**/*.rpm 58 | 59 | publish: 60 | name: "Build and Publish Docker Image" 61 | runs-on: ubuntu-latest 62 | needs: pre-release 63 | steps: 64 | - name: Checkout 65 | uses: actions/checkout@v2 66 | - name: Set up QEMU 67 | uses: docker/setup-qemu-action@v1 68 | - name: Set up Docker Buildx 69 | uses: docker/setup-buildx-action@v1 70 | - name: Login to DockerHub 71 | uses: docker/login-action@v1 72 | with: 73 | username: ${{ secrets.DOCKER_HUB_USER }} 74 | password: ${{ secrets.DOCKER_HUB_PASSWD }} 75 | - name: Build and Push Docker Image 76 | uses: docker/build-push-action@v2 77 | with: 78 | context: . 79 | file: ./Dockerfile 80 | push: true 81 | build-args: | 82 | revision=${{ github.event.inputs.major }}.${{ github.event.inputs.minor }}.${{ github.event.inputs.patch }} 83 | repository=${{ github.repository }} 84 | tags: | 85 | ${{ github.event.inputs.organization }}/jdbc-bridge:latest 86 | ${{ github.event.inputs.organization }}/jdbc-bridge:${{ github.event.inputs.major }} 87 | ${{ github.event.inputs.organization }}/jdbc-bridge:${{ github.event.inputs.major }}.${{ github.event.inputs.minor }} 88 | ${{ github.event.inputs.organization }}/jdbc-bridge:${{ github.event.inputs.major }}.${{ github.event.inputs.minor }}.${{ github.event.inputs.patch }} 89 | 90 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .classpath 3 | .factorypath 4 | .project 5 | dependency-reduced-pom.xml 6 | .settings/ 7 | .vscode/ 8 | demo/ 9 | target/ 10 | test.sh 11 | test-dir/ 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2019-2022, Zhichun Wu 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, 15 | # software distributed under the License is distributed on an 16 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | # KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations 19 | # under the License. 20 | FROM adoptopenjdk/openjdk8-openj9:jre8u322-b06_openj9-0.30.0-ubuntu 21 | 22 | ARG revision=latest 23 | ARG repository=ClickHouse/clickhouse-jdbc-bridge 24 | 25 | # Maintainer 26 | LABEL maintainer="zhicwu@gmail.com" 27 | 28 | # Environment variables 29 | ENV JDBC_BRIDGE_HOME=/app JDBC_BRIDGE_VERSION=${revision} \ 30 | JDBC_BRIDGE_REL_URL=https://github.com/${repository}/releases/download 31 | 32 | # Labels 33 | LABEL app_name="ClickHouse JDBC Bridge" app_version="$JDBC_BRIDGE_VERSION" 34 | 35 | # Update system and install additional packages for troubleshooting 36 | RUN apt-get update \ 37 | && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated apache2-utils \ 38 | apt-transport-https curl htop iftop iptraf iputils-ping jq lsof net-tools tzdata wget \ 39 | && apt-get clean \ 40 | && if [ "$revision" = "latest" ] ; then export JDBC_BRIDGE_VERSION=$(curl -s https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc-bridge/maven-metadata.xml | grep '' | sed -e 's|^.*>\(.*\)<.*$|\1|'); else export JDBC_BRIDGE_VERSION=${revision}; fi \ 41 | && wget -P $JDBC_BRIDGE_HOME/drivers \ 42 | https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc/0.3.2-patch8/clickhouse-jdbc-0.3.2-patch8-all.jar \ 43 | https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/3.0.4/mariadb-java-client-3.0.4.jar \ 44 | https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar \ 45 | https://repo1.maven.org/maven2/org/neo4j/neo4j-jdbc-driver/4.0.5/neo4j-jdbc-driver-4.0.5.jar \ 46 | https://repo1.maven.org/maven2/com/amazon/opendistroforelasticsearch/client/opendistro-sql-jdbc/1.13.0.0/opendistro-sql-jdbc-1.13.0.0.jar \ 47 | https://repo1.maven.org/maven2/org/postgresql/postgresql/42.3.4/postgresql-42.3.4.jar \ 48 | https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.36.0.3/sqlite-jdbc-3.36.0.3.jar \ 49 | https://repo1.maven.org/maven2/io/trino/trino-jdbc/377/trino-jdbc-377.jar \ 50 | && export JDBC_BRIDGE_REL_URL=$JDBC_BRIDGE_REL_URL/v$JDBC_BRIDGE_VERSION \ 51 | && wget -q -P $JDBC_BRIDGE_HOME $JDBC_BRIDGE_REL_URL/LICENSE $JDBC_BRIDGE_REL_URL/NOTICE \ 52 | $JDBC_BRIDGE_REL_URL/clickhouse-jdbc-bridge-${JDBC_BRIDGE_VERSION}-shaded.jar \ 53 | && ln -s $JDBC_BRIDGE_HOME/clickhouse-jdbc-bridge-${JDBC_BRIDGE_VERSION}-shaded.jar $JDBC_BRIDGE_HOME/clickhouse-jdbc-bridge-shaded.jar \ 54 | && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 55 | 56 | COPY --chown=root:root docker/ $JDBC_BRIDGE_HOME 57 | 58 | RUN chmod +x $JDBC_BRIDGE_HOME/*.sh \ 59 | && mkdir -p $JDBC_BRIDGE_HOME/logs /usr/local/lib/java \ 60 | && ln -s $JDBC_BRIDGE_HOME/logs /var/log/clickhouse-jdbc-bridge \ 61 | && ln -s $JDBC_BRIDGE_HOME/clickhouse-jdbc-bridge-${JDBC_BRIDGE_VERSION}-shaded.jar \ 62 | /usr/local/lib/java/clickhouse-jdbc-bridge-shaded.jar \ 63 | && ln -s $JDBC_BRIDGE_HOME /etc/clickhouse-jdbc-bridge 64 | 65 | WORKDIR $JDBC_BRIDGE_HOME 66 | 67 | EXPOSE 9019 68 | 69 | VOLUME ["${JDBC_BRIDGE_HOME}/drivers", "${JDBC_BRIDGE_HOME}/extensions", "${JDBC_BRIDGE_HOME}/logs", "${JDBC_BRIDGE_HOME}/scripts"] 70 | 71 | CMD "./docker-entrypoint.sh" 72 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | clickhouse-jdbc-bridge 2 | Copyright 2019-2021, Zhichun Wu. 3 | 4 | This project includes software developed by Zhichun Wu. 5 | https://github.com/zhicwu 6 | 7 | Licensed under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at: 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, 14 | software distributed under the License is distributed on 15 | an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | KIND, either express or implied. See the License for the 17 | specific language governing permissions and limitations 18 | under the License. 19 | 20 | This project includes: 21 | Apache Commons Compress under Apache License, Version 2.0 22 | Caffeine cache under Apache License, Version 2.0 23 | Checker Qual under The MIT License 24 | ClickHouse JDBC Bridge under Apache License, Version 2.0 25 | dnsjava under BSD-3-Clause 26 | docker-java-api under The Apache Software License, Version 2.0 27 | docker-java-transport under The Apache Software License, Version 2.0 28 | docker-java-transport-zerodep under The Apache Software License, Version 2.0 29 | Duct Tape under MIT 30 | error-prone annotations under Apache 2.0 31 | Hamcrest Core under New BSD License 32 | HdrHistogram under Public Domain, per Creative Commons CC0 or BSD-2-Clause 33 | HikariCP under The Apache Software License, Version 2.0 34 | Jackson-annotations under The Apache Software License, Version 2.0 35 | Jackson-core under The Apache Software License, Version 2.0 36 | jackson-databind under The Apache Software License, Version 2.0 37 | Java Native Access under LGPL, version 2.1 or Apache License v2.0 38 | jcommander under Apache 2.0 39 | JetBrains Java Annotations under The Apache Software License, Version 2.0 40 | JUnit under Eclipse Public License 1.0 41 | LatencyUtils under Public Domain, per Creative Commons CC0 42 | micrometer-core under The Apache Software License, Version 2.0 43 | micrometer-registry-prometheus under The Apache Software License, Version 2.0 44 | Netty/Buffer under Apache License, Version 2.0 45 | Netty/Codec under Apache License, Version 2.0 46 | Netty/Codec/DNS under Apache License, Version 2.0 47 | Netty/Codec/HTTP under Apache License, Version 2.0 48 | Netty/Codec/HTTP2 under Apache License, Version 2.0 49 | Netty/Codec/Socks under Apache License, Version 2.0 50 | Netty/Common under Apache License, Version 2.0 51 | Netty/Handler under Apache License, Version 2.0 52 | Netty/Handler/Proxy under Apache License, Version 2.0 53 | Netty/Resolver under Apache License, Version 2.0 54 | Netty/Resolver/DNS under Apache License, Version 2.0 55 | Netty/TomcatNative [OpenSSL - Classes] under The Apache Software License, Version 2.0 56 | Netty/Transport under Apache License, Version 2.0 57 | Prometheus Java Simpleclient under The Apache Software License, Version 2.0 58 | Prometheus Java Simpleclient Common under The Apache Software License, Version 2.0 59 | SLF4J API Module under MIT License 60 | SLF4J JDK14 Binding under MIT License 61 | Testcontainers Core under MIT 62 | testng under Apache 2.0 63 | Vert.x Bridge Common under The Apache Software License, Version 2.0 or Eclipse Public License - v 1.0 64 | Vert.x Core under The Apache Software License, Version 2.0 or Eclipse Public License - v 2.0 65 | Vert.x metrics implementation for Micrometer.io under The Apache Software License, Version 2.0 or Eclipse Public License - v 1.0 66 | vertx-auth-common under The Apache Software License, Version 2.0 or Eclipse Public License - v 1.0 67 | vertx-config under Apache License, Version 2.0 68 | vertx-web under The Apache Software License, Version 2.0 or Eclipse Public License - v 2.0 69 | vertx-web-client under The Apache Software License, Version 2.0 or Eclipse Public License - v 1.0 70 | vertx-web-common under The Apache Software License, Version 2.0 or Eclipse Public License - v 1.0 71 | 72 | -------------------------------------------------------------------------------- /all-in-one.Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2019-2022, Zhichun Wu 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, 15 | # software distributed under the License is distributed on an 16 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | # KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations 19 | # under the License. 20 | 21 | # docker build --squash --build-arg revision=22.3 -f all-in-one.Dockerfile -t my/clickhouse-all-in-one:22.3 . 22 | ARG revision=22.3 23 | 24 | # 25 | # Stage 1/2: Build 26 | # 27 | FROM maven:3-openjdk-8 as builder 28 | 29 | ARG revision 30 | 31 | COPY LICENSE NOTICE pom.xml /app/ 32 | COPY docker /app/docker/ 33 | COPY misc /app/misc/ 34 | COPY src /app/src/ 35 | 36 | WORKDIR /app 37 | 38 | RUN apt-get update \ 39 | && apt-get install -y --no-install-recommends dpkg fakeroot rpm \ 40 | && mvn -Prelease -DskipTests package 41 | 42 | 43 | # 44 | # Stage 2/2: Pack 45 | # 46 | FROM clickhouse/clickhouse-server:${revision} 47 | 48 | # Maintainer 49 | LABEL maintainer="zhicwu@gmail.com" 50 | 51 | COPY --from=builder /app/target/clickhouse-jdbc-bridge*.deb / 52 | 53 | # DEBIAN_FRONTEND=noninteractive 54 | RUN apt-get update \ 55 | && apt-get install -y --no-install-recommends --allow-unauthenticated \ 56 | apache2-utils apt-transport-https curl htop iftop iptraf iputils-ping jq lsof net-tools tzdata wget \ 57 | && apt-get install -y --no-install-recommends /*.deb \ 58 | && apt-get clean \ 59 | && rm -rf /*.deb /var/lib/apt/lists/* /tmp/* /var/tmp/* \ 60 | && wget -q -P /etc/clickhouse-jdbc-bridge/drivers/clickhouse \ 61 | https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc/0.3.2-patch8/clickhouse-jdbc-0.3.2-patch8-all.jar \ 62 | && wget -q -P /etc/clickhouse-jdbc-bridge/drivers/mariadb \ 63 | https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/3.0.4/mariadb-java-client-3.0.4.jar \ 64 | && wget -q -P /etc/clickhouse-jdbc-bridge/drivers/mysql5 \ 65 | https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar \ 66 | && wget -q -P /etc/clickhouse-jdbc-bridge/drivers/mysql8 \ 67 | https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar \ 68 | && wget -q -P /etc/clickhouse-jdbc-bridge/drivers/postgres \ 69 | https://repo1.maven.org/maven2/org/postgresql/postgresql/42.3.4/postgresql-42.3.4.jar \ 70 | && sed -i -e 's|\(^[[:space:]]*\)\(exec.*clickhouse-server.*$\)|\1exec -c clickhouse-jdbc-bridge >/dev/null \&\n\1\2|' /entrypoint.sh \ 71 | && echo '{\n\ 72 | "$schema": "../datasource-schema.json",\n\ 73 | "self": {\n\ 74 | "driverUrls": [ "drivers/clickhouse" ],\n\ 75 | "driverClassName": "com.clickhouse.jdbc.ClickHouseDriver",\n\ 76 | "jdbcUrl": "jdbc:clickhouse://localhost:8123/system?ssl=false",\n\ 77 | "username": "default",\n\ 78 | "password": "",\n\ 79 | "maximumPoolSize": 5\n\ 80 | }\n\ 81 | }' > /etc/clickhouse-jdbc-bridge/config/datasources/self.json 82 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # ClickHouse JDBC Bridge Docker Image 2 | 3 | ## What is ClickHouse JDBC Bridge? 4 | It's a stateless proxy passing queries from ClickHouse to external datasources. With this extension, you can use [JDBC table function](https://clickhouse.tech/docs/en/sql-reference/table-functions/jdbc/) and/or corresponding [table engine](https://clickhouse.tech/docs/en/engines/table-engines/integrations/jdbc/) on ClickHouse server to query arbitrary datasources in real time. 5 | 6 | ## Versions 7 | The latest tag points to the latest release from `master` branch. Branch tags like `2.0` point to the latest release of the corresponding branch. Full version tags like `2.0.0` point to the corresponding release. 8 | 9 | ## How to use this image 10 | 11 | ### Start JDBC bridge 12 | ```bash 13 | docker run -d --name ch-jdbc-bridge -p9019:9019 \ 14 | -e MAVEN_REPO_URL="https://repo1.maven.org/maven2" \ 15 | -e JDBC_DRIVERS="org/mariadb/jdbc/mariadb-java-client/2.7.4/mariadb-java-client-2.7.4.jar,org/postgresql/postgresql/42.2.24/postgresql-42.2.24.jar" clickhouse/jdbc-bridge 16 | ``` 17 | If you prefer to use JDBC drivers and named datasources on host, you can use the following commands: 18 | ```bash 19 | wget -P drivers \ 20 | https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.7.4/mariadb-java-client-2.7.4.jar \ 21 | https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.24/postgresql-42.2.24.jar 22 | wget -P datasources \ 23 | https://raw.githubusercontent.com/ClickHouse/clickhouse-jdbc-bridge/master/misc/quick-start/jdbc-bridge/config/datasources/mariadb10.json \ 24 | https://raw.githubusercontent.com/ClickHouse/clickhouse-jdbc-bridge/master/misc/quick-start/jdbc-bridge/config/datasources/postgres13.json 25 | # please edit datasources/*.json to connect to your database servers 26 | docker run -d --name ch-jdbc-bridge -p9019:9019 -v `pwd`/drivers:/app/drivers \ 27 | -v `pwd`/datasources:/app/config/datasources clickhouse/jdbc-bridge 28 | ``` 29 | 30 | ### Configure ClickHouse server 31 | By default, ClickHouse assumes JDBC bridge is available at `localhost:9019`. You can customize the host and port in `/etc/clickhouse-server/config.xml` like below: 32 | ```xml 33 | 34 | ... 35 | 36 | localhost 37 | 9019 38 | 39 | ... 40 | 41 | ``` 42 | 43 | ### Issue query on ClickHouse 44 | Once you started JDBC bridge and configured ClickHouse server accordingly, you should be able to run queries like below on ClickHouse: 45 | ```sql 46 | -- show all named datasources 47 | select * from jdbc('', 'show datasources') 48 | -- query against adhoc datasource, NOT recommended for security reason 49 | select * from jdbc('jdbc:mariadb://localhost:3306/test?useSSL=false&user=root&password=root', 'select 1') 50 | -- query against named datasource with inline schema and adhoc query 51 | select * from jdbc('mariadb10', 'num UInt8', 'select 1 as num') 52 | -- scripting 53 | select * from jdbc('script', '[1,2,3]') 54 | ``` 55 | 56 | ## Configuration 57 | 58 | Container exposes 9019 port for ClickHouse integration and monitoring. 59 | 60 | In order to customize the container, please refer to directory structure and supported environment variables shown below. 61 | ```bash 62 | /app # work directory 63 | | 64 | |-- drivers # JDBC drivers 65 | |-- config 66 | | | 67 | | |-- datasources # named datasources 68 | | |-- schemas # named schemas 69 | | |-- queries # named queries 70 | | 71 | |-- logs # application logs 72 | |-- scripts # saved queries/scripts 73 | ``` 74 | 75 | Environment Variable | Java System Property | Default Value | Remark 76 | -- | -- | -- | -- 77 | CONFIG_DIR | jdbc-bridge.config.dir | config | Configuration directory 78 | SERIAL_MODE | jdbc-bridge.serial.mode | false | Whether run query in serial mode or not 79 | CUSTOM_DRIVER_LOADER | jdbc-bridge.driver.loader | true | Whether use custom driver class loader or not 80 | DATASOURCE_CONFIG_DIR | jdbc-bridge.datasource.config.dir | datasources | Directory for named datasources 81 | DEFAULT_VALUE | jdbc-bridge.type.default | false | Whether support default expression in column definition or not 82 | DRIVER_DIR | jdbc-bridge.driver.dir | drivers | Directory for drivers needed to connect to datasources 83 | HTTPD_CONFIG_FILE | jdbc-bridge.httpd.config.file | httpd.json | HTTP server configuration file 84 | JDBC_BRIDGE_JVM_OPTS | - | - | JVM arguments 85 | JDBC_DRIVERS | - | - | Comma separated JDBC driver download path on Maven repository 86 | MAVEN_REPO_URL | - | https://repo1.maven.org/maven2 | Base URL of Maven repository 87 | QUERY_CONFIG_DIR | jdbc-bridge.query.config.dir | queries | Directory for named queries 88 | SCHEMA_CONFIG_DIR | jdbc-bridge.schema.config.dir | schemas | Directory for named schemas 89 | SERVER_CONFIG_FILE | jdbc-bridge.server.config.file | server.json | JDBC bridge server configuration file 90 | VERTX_CONFIG_FILE | jdbc-bridge.vertx.config.file | vertx.json | Vert.x configuration file 91 | 92 | 93 | ## License 94 | 95 | View [license information](https://github.com/ClickHouse/clickhouse-jdbc-bridge/blob/master/LICENSE) for the software contained in this image. 96 | -------------------------------------------------------------------------------- /docker/config/datasources/datasource.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../datasource.jschema", 3 | "ch-server": { 4 | "aliases": [ 5 | "self" 6 | ], 7 | "driverUrls": [ 8 | "https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc/0.3.2-patch8/clickhouse-jdbc-0.3.2-patch8-http.jar" 9 | ], 10 | "driverClassName": "com.clickhouse.jdbc.ClickHouseDriver", 11 | "jdbcUrl": "jdbc:clickhouse://ch-server:8123/system?compress=false", 12 | "username": "default", 13 | "password": "", 14 | "maximumPoolSize": 5 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /docker/config/datasources/script.json: -------------------------------------------------------------------------------- 1 | { 2 | "script": { 3 | "type": "script", 4 | "driverUrls": [ 5 | "/d/Tools/groovy-3.0.7/lib", 6 | "!/d/Tools/groovy-3.0.7/lib/groovy-jaxb-3.0.7.jar", 7 | "D:/Tools/groovy-3.0.7/lib", 8 | "!D:/Tools/groovy-3.0.7/lib/groovy-jaxb-3.0.7.jar" 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /docker/config/httpd.json: -------------------------------------------------------------------------------- 1 | { 2 | "maxInitialLineLength": 2147483647 3 | } 4 | -------------------------------------------------------------------------------- /docker/config/queries/query.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../query-schema.json", 3 | "show-query-logs": { 4 | "query": "scripts/show-query-logs.sql", 5 | "columns": [ 6 | { 7 | "name": "query_id", 8 | "type": "String", 9 | "nullable": false 10 | }, 11 | { 12 | "name": "type", 13 | "type": "String", 14 | "nullable": false 15 | }, 16 | { 17 | "name": "event_time", 18 | "type": "DateTime64", 19 | "nullable": false, 20 | "scale": 3 21 | }, 22 | { 23 | "name": "query_start_time", 24 | "type": "DateTime64", 25 | "nullable": false, 26 | "scale": 3 27 | }, 28 | { 29 | "name": "query", 30 | "type": "String", 31 | "nullable": false 32 | }, 33 | { 34 | "name": "user", 35 | "type": "String", 36 | "nullable": false 37 | } 38 | ], 39 | "parameters": { 40 | "fetch_size": 100, 41 | "max_rows": 100 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /docker/config/query.jschema: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "definitions": { 4 | "column": { 5 | "description": "Column.", 6 | "type": "object", 7 | "properties": { 8 | "name": { 9 | "description": "Column name.", 10 | "type": "string" 11 | }, 12 | "type": { 13 | "description": "Data type.", 14 | "type": "string", 15 | "enum": [ 16 | "Boolean", 17 | "Int8", 18 | "Int16", 19 | "Int32", 20 | "Int64", 21 | "Int128", 22 | "Int256", 23 | "UInt8", 24 | "UInt16", 25 | "UInt32", 26 | "UInt64", 27 | "UInt128", 28 | "UInt256", 29 | "Float32", 30 | "Float64", 31 | "Date", 32 | "DateTime", 33 | "DateTime64", 34 | "Decimal", 35 | "Decimal32", 36 | "Decimal64", 37 | "Decimal128", 38 | "Decimal256", 39 | "Enum", 40 | "Enum8", 41 | "Enum16", 42 | "IPv4", 43 | "IPv6", 44 | "FixedString", 45 | "String", 46 | "UUID" 47 | ] 48 | }, 49 | "value": { 50 | "default": "", 51 | "description": "Constant value in string format.", 52 | "type": "string" 53 | }, 54 | "nullable": { 55 | "default": true, 56 | "description": "Whether the columns may contain null value or not.", 57 | "type": "boolean" 58 | }, 59 | "precision": { 60 | "default": 0, 61 | "description": "Precision of decimal value.", 62 | "type": "integer" 63 | }, 64 | "scale": { 65 | "default": 0, 66 | "description": "Scale of decimal value.", 67 | "type": "integer" 68 | } 69 | } 70 | }, 71 | "columns": { 72 | "description": "List of columns.", 73 | "type": "array", 74 | "items": { 75 | "$ref": "#/definitions/column" 76 | }, 77 | "minItems": 1 78 | }, 79 | "parameters": { 80 | "description": "Query parameters.", 81 | "type": "object", 82 | "properties": { 83 | "fetch_size": { 84 | "default": 1000, 85 | "description": "JDBC fetch size, defaults to 1000.", 86 | "type": "integer" 87 | }, 88 | "max_rows": { 89 | "default": 0, 90 | "description": "Max rows to read from result sets, defaults to 0, meaning all.", 91 | "type": "integer" 92 | }, 93 | "null_as_default": { 94 | "default": false, 95 | "description": "Whether to replace null value to default value of corresponding type.", 96 | "type": "boolean" 97 | }, 98 | "offset": { 99 | "default": 0, 100 | "description": "Offset, or skip the first n rows, to start reading from result set.", 101 | "type": "integer" 102 | }, 103 | "position": { 104 | "default": 0, 105 | "description": "Absolute position of cursor to start reading from result set. 1 for first row and -1 for the last.", 106 | "type": "integer" 107 | }, 108 | "datasource_column": { 109 | "default": false, 110 | "description": "Whether to display datasource column in query result.", 111 | "type": "boolean" 112 | }, 113 | "custom_columns": { 114 | "default": false, 115 | "description": "Whether to display customized columns(defined in datasource configuration) in query results.", 116 | "type": "boolean" 117 | } 118 | } 119 | }, 120 | "entity": { 121 | "description": "Named query.", 122 | "type": "object", 123 | "properties": { 124 | "query": { 125 | "description": "The query.", 126 | "type": "string", 127 | "examples": [ 128 | "some_table", 129 | "select * from some_table", 130 | "scripts/query_some_table.sql" 131 | ] 132 | }, 133 | "schema": { 134 | "description": "Named schema which overrides columns.", 135 | "type": "string" 136 | }, 137 | "columns": { 138 | "$ref": "#/definitions/columns" 139 | }, 140 | "parameters": { 141 | "$ref": "#/definitions/parameters" 142 | } 143 | }, 144 | "required": [ 145 | "query", 146 | "columns" 147 | ] 148 | } 149 | }, 150 | "type": "object", 151 | "additionalProperties": { 152 | "$ref": "#/definitions/entity" 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /docker/config/schema.jschema: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "definitions": { 4 | "column": { 5 | "description": "Column.", 6 | "type": "object", 7 | "properties": { 8 | "name": { 9 | "description": "Column name.", 10 | "type": "string" 11 | }, 12 | "type": { 13 | "description": "Data type.", 14 | "type": "string", 15 | "enum": [ 16 | "Boolean", 17 | "Int8", 18 | "Int16", 19 | "Int32", 20 | "Int64", 21 | "Int128", 22 | "Int256", 23 | "UInt8", 24 | "UInt16", 25 | "UInt32", 26 | "UInt64", 27 | "UInt128", 28 | "UInt256", 29 | "Float32", 30 | "Float64", 31 | "Date", 32 | "DateTime", 33 | "DateTime64", 34 | "Decimal", 35 | "Decimal32", 36 | "Decimal64", 37 | "Decimal128", 38 | "Decimal256", 39 | "Enum", 40 | "Enum8", 41 | "Enum16", 42 | "IPv4", 43 | "IPv6", 44 | "FixedString", 45 | "String", 46 | "UUID" 47 | ] 48 | }, 49 | "value": { 50 | "default": "", 51 | "description": "Constant value in string format.", 52 | "type": "string" 53 | }, 54 | "nullable": { 55 | "default": true, 56 | "description": "Whether the columns may contain null value or not.", 57 | "type": "boolean" 58 | }, 59 | "precision": { 60 | "default": 0, 61 | "description": "Precision of decimal value.", 62 | "type": "integer" 63 | }, 64 | "scale": { 65 | "default": 0, 66 | "description": "Scale of decimal value.", 67 | "type": "integer" 68 | } 69 | } 70 | }, 71 | "columns": { 72 | "description": "List of columns.", 73 | "type": "array", 74 | "items": { 75 | "$ref": "#/definitions/column" 76 | }, 77 | "minItems": 1 78 | }, 79 | "entity": { 80 | "description": "Named schema.", 81 | "type": "object", 82 | "properties": { 83 | "columns": { 84 | "$ref": "#/definitions/columns" 85 | } 86 | }, 87 | "required": [ 88 | "columns" 89 | ] 90 | } 91 | }, 92 | "type": "object", 93 | "additionalProperties": { 94 | "$ref": "#/definitions/entity" 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /docker/config/schemas/schema.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../schema.jschema", 3 | "query-log": { 4 | "columns": [ 5 | { 6 | "name": "query_id", 7 | "type": "String", 8 | "nullable": false 9 | }, 10 | { 11 | "name": "type", 12 | "type": "String", 13 | "nullable": false 14 | }, 15 | { 16 | "name": "event_time", 17 | "type": "DateTime64", 18 | "nullable": false, 19 | "scale": 3 20 | }, 21 | { 22 | "name": "query_start_time", 23 | "type": "DateTime64", 24 | "nullable": false, 25 | "scale": 3 26 | }, 27 | { 28 | "name": "query", 29 | "type": "String", 30 | "nullable": false 31 | }, 32 | { 33 | "name": "user", 34 | "type": "String", 35 | "nullable": false 36 | } 37 | ] 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /docker/config/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestTimeout": 5000, 3 | "queryTimeout": 60000, 4 | "configScanPeriod": 5000, 5 | "repositories": [ 6 | { 7 | "entity": "com.clickhouse.jdbcbridge.core.NamedDataSource", 8 | "repository": "com.clickhouse.jdbcbridge.impl.JsonFileRepository" 9 | }, 10 | { 11 | "entity": "com.clickhouse.jdbcbridge.core.NamedSchema", 12 | "repository": "com.clickhouse.jdbcbridge.impl.JsonFileRepository" 13 | }, 14 | { 15 | "entity": "com.clickhouse.jdbcbridge.core.NamedQuery", 16 | "repository": "com.clickhouse.jdbcbridge.impl.JsonFileRepository" 17 | } 18 | ], 19 | "extensions": [ 20 | { 21 | "class": "com.clickhouse.jdbcbridge.impl.JdbcDataSource" 22 | }, 23 | { 24 | "class": "com.clickhouse.jdbcbridge.impl.ConfigDataSource" 25 | }, 26 | { 27 | "class": "com.clickhouse.jdbcbridge.impl.ScriptDataSource" 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /docker/config/vertx.json: -------------------------------------------------------------------------------- 1 | { 2 | "maxEventLoopExecuteTime": 15, 3 | "maxEventLoopExecuteTimeUnit": "SECONDS", 4 | "maxWorkerExecuteTime": 300, 5 | "maxWorkerExecuteTimeUnit": "SECONDS", 6 | "workerPoolSize": 20 7 | } 8 | -------------------------------------------------------------------------------- /docker/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | start_server() { 6 | local base_url="${MAVEN_REPO_URL:="https://repo1.maven.org/maven2"}" 7 | local driver_dir="$JDBC_BRIDGE_HOME/drivers" 8 | local jdbc_drivers="${JDBC_DRIVERS:=""}" 9 | 10 | # change work directory explicitly 11 | cd $JDBC_BRIDGE_HOME 12 | 13 | if [ "$jdbc_drivers" != "" ] && [ "$(ls -A $driver_dir)" == "" ]; then 14 | echo "Downloading JDBC drivers to directory [$driver_dir]..." 15 | for i in $(echo "$jdbc_drivers" | sed "s/,/ /g"); do 16 | if [ "$i" != "" ]; then 17 | echo " => [$base_url/$i]..." 18 | wget -q -P "$driver_dir" "$base_url/$i" 19 | fi 20 | done 21 | fi 22 | 23 | if [ "$(echo ${CUSTOM_DRIVER_LOADER:="true"} | tr '[:upper:]' '[:lower:]')" != "true" ]; then 24 | local classpath="./clickhouse-jdbc-bridge-shaded.jar:$(echo $(ls ${DRIVER_DIR:="drivers"}/*.jar) | tr ' ' ':'):." 25 | java -XX:+UseContainerSupport -XX:+IdleTuningCompactOnIdle -XX:+IdleTuningGcOnIdle \ 26 | -Xdump:none -Xdump:tool:events=systhrow+throw,filter=*OutOfMemoryError,exec="kill -9 %pid" \ 27 | -Djava.util.logging.config.file=$JDBC_BRIDGE_HOME/logging.properties -Dnashorn.args=--language=es6 \ 28 | ${JDBC_BRIDGE_JVM_OPTS:=""} -cp $classpath com.clickhouse.jdbcbridge.JdbcBridgeVerticle 29 | else 30 | java -XX:+UseContainerSupport -XX:+IdleTuningCompactOnIdle -XX:+IdleTuningGcOnIdle \ 31 | -Xdump:none -Xdump:tool:events=systhrow+throw,filter=*OutOfMemoryError,exec="kill -9 %pid" \ 32 | -Djava.util.logging.config.file=$JDBC_BRIDGE_HOME/logging.properties -Dnashorn.args=--language=es6 \ 33 | ${JDBC_BRIDGE_JVM_OPTS:=""} -jar clickhouse-jdbc-bridge-shaded.jar 34 | fi 35 | } 36 | 37 | if [ $# -eq 0 ]; then 38 | start_server 39 | else 40 | exec "$@" 41 | fi 42 | -------------------------------------------------------------------------------- /docker/logging.properties: -------------------------------------------------------------------------------- 1 | handlers=java.util.logging.FileHandler, java.util.logging.ConsoleHandler 2 | 3 | java.util.logging.ConsoleHandler.level=DEBUG 4 | java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter 5 | java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$-7s] %5$s %n 6 | 7 | java.util.logging.FileHandler.level=INFO 8 | java.util.logging.FileHandler.pattern=/var/log/clickhouse-jdbc-bridge/console.log 9 | java.util.logging.FileHandler.limit=10000000 10 | java.util.logging.FileHandler.count=5 11 | java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter 12 | java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$-7s] %5$s %n 13 | 14 | .level=DEBUG 15 | -------------------------------------------------------------------------------- /docker/scripts/one-two-three.js: -------------------------------------------------------------------------------- 1 | "1,2,3".split(",").map(Number) 2 | -------------------------------------------------------------------------------- /docker/scripts/show-query-logs.sql: -------------------------------------------------------------------------------- 1 | select * from system.query_log 2 | -------------------------------------------------------------------------------- /docker/set-env.sh.example: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | JVM_ARGS="-Xms1024m -Xmx2048m" 4 | -------------------------------------------------------------------------------- /misc/license-header.template: -------------------------------------------------------------------------------- 1 | Copyright ${inceptionYear}-${latestYearOfContribution}, Zhichun Wu 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /misc/notice.template: -------------------------------------------------------------------------------- 1 | clickhouse-jdbc-bridge 2 | Copyright 2019-2021, Zhichun Wu. 3 | 4 | This project includes software developed by Zhichun Wu. 5 | https://github.com/zhicwu 6 | 7 | Licensed under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at: 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, 14 | software distributed under the License is distributed on 15 | an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | KIND, either express or implied. See the License for the 17 | specific language governing permissions and limitations 18 | under the License. 19 | 20 | This project includes: 21 | #GENERATED_NOTICES# -------------------------------------------------------------------------------- /misc/perf-test/clickhouse/config/perf_test_config.xml: -------------------------------------------------------------------------------- 1 | 2 | 0.0.0.0 3 | 3307 4 | 5 | jdbc-bridge 6 | 9019 7 | 8 | 9 | /metrics 10 | 8001 11 | true 12 | true 13 | true 14 | 15 | -------------------------------------------------------------------------------- /misc/perf-test/clickhouse/init.sql: -------------------------------------------------------------------------------- 1 | DROP VIEW IF EXISTS system.constant; 2 | CREATE VIEW system.constant AS SELECT 1; 3 | 4 | DROP TABLE IF EXISTS system.10k_rows; 5 | CREATE TABLE system.10k_rows 6 | ( 7 | `id` Int64, 8 | `name` String, 9 | `datetime` DateTime, 10 | `num` Nullable(Int64), 11 | `value` Nullable(Float32) 12 | ) ENGINE = MergeTree() PARTITION BY toYYYYMM(datetime) ORDER BY (id) SETTINGS index_granularity = 8192; 13 | INSERT INTO 10k_rows SELECT * FROM jdbc('mariadb', '10k_rows'); -------------------------------------------------------------------------------- /misc/perf-test/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | # 4 | # perf-test1, perf-test2, and perf-test3 are KVMs running CentOS 7 5 | # 6 | services: 7 | clickhouse: 8 | image: clickhouse/clickhouse-server:22.3 9 | hostname: ch-server 10 | ports: 11 | - "3307:3307" 12 | - "8001:8001" 13 | - "8123:8123" 14 | - "9000:9000" 15 | entrypoint: /bin/bash 16 | command: > 17 | -c " 18 | echo ' 19 | 0.0.0.0 20 | 3307 21 | 22 | jdbc-bridge 23 | 9019 24 | 25 | 26 | /metrics 27 | 8001 28 | true 29 | true 30 | true 31 | 32 | ' > /etc/clickhouse-server/config.d/perf_test_config.xml && /entrypoint.sh 33 | " 34 | deploy: 35 | mode: replicated 36 | replicas: 1 37 | resources: 38 | reservations: 39 | memory: 4196M 40 | limits: 41 | memory: 8192M 42 | placement: 43 | constraints: 44 | - node.labels.alias == perf-test2 45 | update_config: 46 | parallelism: 1 47 | delay: 5s 48 | restart_policy: 49 | condition: on-failure 50 | delay: 5s 51 | max_attempts: 3 52 | window: 60s 53 | 54 | mariadb: 55 | image: mariadb:10.5 56 | hostname: mariadb 57 | ports: 58 | - "3306:3306" 59 | environment: 60 | MYSQL_DATABASE: test 61 | MYSQL_ROOT_PASSWORD: root 62 | deploy: 63 | mode: replicated 64 | replicas: 1 65 | resources: 66 | reservations: 67 | memory: 1024M 68 | limits: 69 | memory: 2048M 70 | placement: 71 | constraints: 72 | - node.labels.alias == perf-test1 73 | update_config: 74 | parallelism: 1 75 | delay: 5s 76 | restart_policy: 77 | condition: on-failure 78 | delay: 5s 79 | max_attempts: 3 80 | window: 60s 81 | 82 | jdbc-bridge: 83 | image: clickhouse/jdbc-bridge 84 | hostname: jdbc-bridge 85 | ports: 86 | - "9019:9019" 87 | command: > 88 | /bin/bash -cx ' 89 | export TEST_CONF_URL="https://raw.githubusercontent.com/ClickHouse/clickhouse-jdbc-bridge/develop/misc/perf-test/jdbc-bridge"; 90 | mkdir -p /app/config/{datasources,queries,schemas} /app/drivers/mariadb /app/scripts; 91 | curl -sSL -o /app/config/datasources/mariadb.json $$TEST_CONF_URL/config/datasources/mariadb.json; 92 | curl -sSL -o /app/config/queries/constant.json $$TEST_CONF_URL/config/queries/constant.json; 93 | curl -sSL -o /app/config/queries/small-table.json $$TEST_CONF_URL/config/queries/small-table.json; 94 | curl -sSL -o /app/config/schemas/simple-num.json $$TEST_CONF_URL/config/schemas/simple-num.json; 95 | curl -sSL -o /app/config/schemas/simple-row.json $$TEST_CONF_URL/config/schemas/simple-row.json; 96 | curl -sSL -o /app/scripts/constant.sql $$TEST_CONF_URL/scripts/constant.sql; 97 | curl -sSL -o /app/scripts/small-table.sql $$TEST_CONF_URL/scripts/small-table.sql; 98 | ./docker-entrypoint.sh 99 | ' 100 | # enable ES6 support in Nashorn 101 | environment: 102 | JDBC_BRIDGE_JVM_OPTS: "-Dnashorn.args=--language=es6" 103 | deploy: 104 | mode: replicated 105 | # docker service scale =2 106 | replicas: 1 107 | resources: 108 | reservations: 109 | memory: 1024M 110 | limits: 111 | memory: 2048M 112 | placement: 113 | constraints: 114 | - node.labels.alias == perf-test3 115 | update_config: 116 | parallelism: 1 117 | delay: 5s 118 | restart_policy: 119 | condition: on-failure 120 | delay: 5s 121 | max_attempts: 3 122 | window: 60s 123 | -------------------------------------------------------------------------------- /misc/perf-test/jdbc-bridge/config/datasources/mariadb.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/datasource.jschema", 3 | "mariadb": { 4 | "driverUrls": [ 5 | "https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.7.4/mariadb-java-client-2.7.4.jar" 6 | ], 7 | "driverClassName": "org.mariadb.jdbc.Driver", 8 | "jdbcUrl": "jdbc:mariadb://mariadb:3306/test?useSSL=false&useCompression=false&rewriteBatchedStatements=true", 9 | "dataSource": { 10 | "user": "root", 11 | "password": "root" 12 | }, 13 | "initializationFailTimeout": 0, 14 | "minimumIdle": 0, 15 | "maximumPoolSize": 20, 16 | "parameters": { 17 | "fetch_size": 10000 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /misc/perf-test/jdbc-bridge/config/queries/constant.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/query.jschema", 3 | "constant": { 4 | "query": "scripts/constant.sql", 5 | "columns": [ 6 | { 7 | "name": "1", 8 | "type": "UInt8", 9 | "nullable": false 10 | } 11 | ] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /misc/perf-test/jdbc-bridge/config/queries/small-table.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/query.jschema", 3 | "small-table": { 4 | "query": "scripts/small-table.sql", 5 | "columns": [ 6 | { "name": "id", "type": "Int64", "nullable": false }, 7 | { "name": "name", "type": "String", "nullable": false }, 8 | { 9 | "name": "datetime", 10 | "type": "DateTime64", 11 | "nullable": true, 12 | "scale": 3 13 | }, 14 | { "name": "num", "type": "Int32", "nullable": true }, 15 | { "name": "value", "type": "Float32", "nullable": true } 16 | ] 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /misc/perf-test/jdbc-bridge/config/schemas/simple-num.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/schema.jschema", 3 | "simple-num": { 4 | "columns": [ 5 | { 6 | "name": "1", 7 | "type": "UInt8", 8 | "nullable": false 9 | } 10 | ] 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /misc/perf-test/jdbc-bridge/config/schemas/simple-row.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/schema.jschema", 3 | "simple-row": { 4 | "columns": [ 5 | { "name": "id", "type": "Int64", "nullable": false }, 6 | { "name": "name", "type": "String", "nullable": false }, 7 | { 8 | "name": "datetime", 9 | "type": "DateTime64", 10 | "nullable": true, 11 | "scale": 3 12 | }, 13 | { "name": "num", "type": "Int32", "nullable": true }, 14 | { "name": "value", "type": "Float32", "nullable": true } 15 | ] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /misc/perf-test/jdbc-bridge/scripts/constant.sql: -------------------------------------------------------------------------------- 1 | select 1 -------------------------------------------------------------------------------- /misc/perf-test/jdbc-bridge/scripts/small-table.sql: -------------------------------------------------------------------------------- 1 | select * from 10k_rows -------------------------------------------------------------------------------- /misc/perf-test/mariadb/init.sql: -------------------------------------------------------------------------------- 1 | DROP VIEW IF EXISTS test.constant; 2 | CREATE VIEW test.constant AS SELECT 1; 3 | 4 | DROP TABLE IF EXISTS `10k_rows`; 5 | CREATE TABLE `10k_rows` 6 | ( 7 | `id` bigint(20) NOT NULL AUTO_INCREMENT, 8 | `name` varchar(50) NOT NULL, 9 | `datetime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, 10 | `num` int(11) DEFAULT NULL, 11 | `value` float DEFAULT NULL, 12 | PRIMARY KEY (`id`) 13 | ); 14 | 15 | DROP PROCEDURE IF EXISTS generate_10k_rows; 16 | DELIMITER $$ 17 | CREATE PROCEDURE generate_10k_rows() 18 | BEGIN 19 | DECLARE i INT DEFAULT 0; 20 | WHILE i < 10000 DO 21 | INSERT INTO `10k_rows` (`name`, `datetime`,`num`, `value`) VALUES ( 22 | FROM_UNIXTIME(UNIX_TIMESTAMP('2020-01-01 00:00:00') + FLOOR(RAND() * 31536000)), 23 | FROM_UNIXTIME(UNIX_TIMESTAMP('2020-01-01 00:00:00') + FLOOR(RAND() * 31536000)), 24 | FLOOR(RAND() * 10000), 25 | ROUND(RAND() * 100, 4) 26 | ); 27 | SET i = i + 1; 28 | END WHILE; 29 | END$$ 30 | DELIMITER ; 31 | 32 | CALL generate_10k_rows(); 33 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse(patched)_10k-rows-query(jdbc).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20jdbc%28%27mariadb%27%2C%27small-table%27%29 13 | Document Length: 575631 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 1186.422 seconds 17 | Complete requests: 100000 18 | Failed requests: 6632 19 | (Connect: 0, Receive: 0, Length: 6632, Exceptions: 0) 20 | Write errors: 0 21 | Non-2xx responses: 6632 22 | Total transferred: 53787967224 bytes 23 | HTML transferred: 53746330944 bytes 24 | Requests per second: 84.29 [#/sec] (mean) 25 | Time per request: 237.284 [ms] (mean) 26 | Time per request: 11.864 [ms] (mean, across all concurrent requests) 27 | Transfer rate: 44273.70 [Kbytes/sec] received 28 | 29 | Connection Times (ms) 30 | min mean[+/-sd] median max 31 | Connect: 0 2 3.5 1 1060 32 | Processing: 61 236 210.3 182 2019 33 | Waiting: 49 204 217.1 148 2019 34 | Total: 61 237 210.1 184 2021 35 | 36 | Percentage of the requests served within a certain time (ms) 37 | 50% 184 38 | 66% 203 39 | 75% 218 40 | 80% 228 41 | 90% 265 42 | 95% 1005 43 | 98% 1006 44 | 99% 1006 45 | 100% 2021 (longest request) 46 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse(patched)_10k-rows-query(jdbc-dual).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20jdbc%28%27mariadb%27%2C%27small-table%27%29 13 | Document Length: 575631 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 1080.676 seconds 17 | Complete requests: 100000 18 | Failed requests: 4195 19 | (Connect: 0, Receive: 0, Length: 4195, Exceptions: 0) 20 | Write errors: 0 21 | Non-2xx responses: 4195 22 | Total transferred: 55190687378 bytes 23 | HTML transferred: 55148843953 bytes 24 | Requests per second: 92.53 [#/sec] (mean) 25 | Time per request: 216.135 [ms] (mean) 26 | Time per request: 10.807 [ms] (mean, across all concurrent requests) 27 | Transfer rate: 49873.55 [Kbytes/sec] received 28 | 29 | Connection Times (ms) 30 | min mean[+/-sd] median max 31 | Connect: 0 2 1.1 1 11 32 | Processing: 62 214 171.2 179 2012 33 | Waiting: 48 183 175.7 147 2012 34 | Total: 65 216 171.0 180 2013 35 | 36 | Percentage of the requests served within a certain time (ms) 37 | 50% 180 38 | 66% 200 39 | 75% 214 40 | 80% 223 41 | 90% 253 42 | 95% 298 43 | 98% 1005 44 | 99% 1006 45 | 100% 2013 (longest request) 46 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse(patched)_constant-query(jdbc).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20jdbc%28%27mariadb%27%2C%27constant%27%29 13 | Document Length: 2 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 833.892 seconds 17 | Complete requests: 100000 18 | Failed requests: 1577 19 | (Connect: 0, Receive: 0, Length: 1577, Exceptions: 0) 20 | Write errors: 0 21 | Non-2xx responses: 1577 22 | Total transferred: 42456772 bytes 23 | HTML transferred: 390817 bytes 24 | Requests per second: 119.92 [#/sec] (mean) 25 | Time per request: 166.778 [ms] (mean) 26 | Time per request: 8.339 [ms] (mean, across all concurrent requests) 27 | Transfer rate: 49.72 [Kbytes/sec] received 28 | 29 | Connection Times (ms) 30 | min mean[+/-sd] median max 31 | Connect: 0 101 306.5 1 3091 32 | Processing: 9 65 121.8 48 2039 33 | Waiting: 9 64 121.8 48 2038 34 | Total: 10 167 325.7 51 3109 35 | 36 | Percentage of the requests served within a certain time (ms) 37 | 50% 51 38 | 66% 56 39 | 75% 60 40 | 80% 64 41 | 90% 1012 42 | 95% 1088 43 | 98% 1096 44 | 99% 1103 45 | 100% 3109 (longest request) 46 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse(patched)_constant-query(jdbc-dual).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20jdbc%28%27mariadb%27%2C%27constant%27%29 13 | Document Length: 2 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 846.403 seconds 17 | Complete requests: 100000 18 | Failed requests: 3021 19 | (Connect: 0, Receive: 0, Length: 3021, Exceptions: 0) 20 | Write errors: 0 21 | Non-2xx responses: 3021 22 | Total transferred: 42508757 bytes 23 | HTML transferred: 565542 bytes 24 | Requests per second: 118.15 [#/sec] (mean) 25 | Time per request: 169.281 [ms] (mean) 26 | Time per request: 8.464 [ms] (mean, across all concurrent requests) 27 | Transfer rate: 49.05 [Kbytes/sec] received 28 | 29 | Connection Times (ms) 30 | min mean[+/-sd] median max 31 | Connect: 0 92 293.8 1 1067 32 | Processing: 8 77 166.9 47 2026 33 | Waiting: 7 76 167.0 47 2026 34 | Total: 8 169 342.4 50 3054 35 | 36 | Percentage of the requests served within a certain time (ms) 37 | 50% 50 38 | 66% 56 39 | 75% 61 40 | 80% 67 41 | 90% 1006 42 | 95% 1086 43 | 98% 1097 44 | 99% 1109 45 | 100% 3054 (longest request) 46 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_10k-rows-query(jdbc).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20jdbc%28%27mariadb%27%2C%27small-table%27%29 13 | Document Length: 575631 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 1483.565 seconds 17 | Complete requests: 100000 18 | Failed requests: 11588 19 | (Connect: 0, Receive: 0, Length: 11588, Exceptions: 0) 20 | Write errors: 0 21 | Non-2xx responses: 11588 22 | Total transferred: 50935481281 bytes 23 | HTML transferred: 50894275862 bytes 24 | Requests per second: 67.41 [#/sec] (mean) 25 | Time per request: 296.713 [ms] (mean) 26 | Time per request: 14.836 [ms] (mean, across all concurrent requests) 27 | Transfer rate: 33528.48 [Kbytes/sec] received 28 | 29 | Connection Times (ms) 30 | min mean[+/-sd] median max 31 | Connect: 0 1 1.1 1 11 32 | Processing: 65 295 264.2 205 2050 33 | Waiting: 56 269 272.5 174 2050 34 | Total: 66 297 264.0 206 2051 35 | 36 | Percentage of the requests served within a certain time (ms) 37 | 50% 206 38 | 66% 230 39 | 75% 250 40 | 80% 266 41 | 90% 1006 42 | 95% 1008 43 | 98% 1011 44 | 99% 1017 45 | 100% 2051 (longest request) 46 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_10k-rows-query(mysql).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20mysql%28%27mariadb%3A3306%27%2C%20%27test%27%2C%20%2710k_rows%27%2C%20%27root%27%2C%20%27root%27%29 13 | Document Length: 575631 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 1657.425 seconds 17 | Complete requests: 100000 18 | Failed requests: 0 19 | Write errors: 0 20 | Total transferred: 57605300000 bytes 21 | HTML transferred: 57563100000 bytes 22 | Requests per second: 60.33 [#/sec] (mean) 23 | Time per request: 331.485 [ms] (mean) 24 | Time per request: 16.574 [ms] (mean, across all concurrent requests) 25 | Transfer rate: 33941.32 [Kbytes/sec] received 26 | 27 | Connection Times (ms) 28 | min mean[+/-sd] median max 29 | Connect: 0 1 1.0 1 13 30 | Processing: 27 330 458.5 121 2227 31 | Waiting: 20 305 461.8 93 2169 32 | Total: 28 331 458.3 123 2228 33 | 34 | Percentage of the requests served within a certain time (ms) 35 | 50% 123 36 | 66% 173 37 | 75% 218 38 | 80% 1043 39 | 90% 1072 40 | 95% 1101 41 | 98% 2055 42 | 99% 2070 43 | 100% 2228 (longest request) 44 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_10k-rows-query(remote).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20remote%28%27ch-server%3A9000%27%2C%20system.10k_rows%2C%20%27default%27%2C%20%27%27%29 13 | Document Length: 575631 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 854.610 seconds 17 | Complete requests: 100000 18 | Failed requests: 0 19 | Write errors: 0 20 | Total transferred: 57605300000 bytes 21 | HTML transferred: 57563100000 bytes 22 | Requests per second: 117.01 [#/sec] (mean) 23 | Time per request: 170.922 [ms] (mean) 24 | Time per request: 8.546 [ms] (mean, across all concurrent requests) 25 | Transfer rate: 65825.57 [Kbytes/sec] received 26 | 27 | Connection Times (ms) 28 | min mean[+/-sd] median max 29 | Connect: 0 80 269.4 4 1067 30 | Processing: 11 91 14.8 94 202 31 | Waiting: 5 13 5.4 12 125 32 | Total: 12 171 257.3 99 1201 33 | 34 | Percentage of the requests served within a certain time (ms) 35 | 50% 99 36 | 66% 101 37 | 75% 102 38 | 80% 104 39 | 90% 113 40 | 95% 1084 41 | 98% 1093 42 | 99% 1098 43 | 100% 1201 (longest request) 44 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_10k-rows-query(url).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20url%28%27http%3A%2F%2Fch-server%3A8123%2F%3Fquery%3Dselect%2520%252A%2520from%2520system.10k_rows%27%2C%20CSV%2C%20%27results%20String%27%29 13 | Document Length: 615631 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 853.292 seconds 17 | Complete requests: 100000 18 | Failed requests: 5 19 | (Connect: 0, Receive: 0, Length: 5, Exceptions: 0) 20 | Write errors: 0 21 | Non-2xx responses: 5 22 | Total transferred: 61602222130 bytes 23 | HTML transferred: 61560022555 bytes 24 | Requests per second: 117.19 [#/sec] (mean) 25 | Time per request: 170.658 [ms] (mean) 26 | Time per request: 8.533 [ms] (mean, across all concurrent requests) 27 | Transfer rate: 70501.57 [Kbytes/sec] received 28 | 29 | Connection Times (ms) 30 | min mean[+/-sd] median max 31 | Connect: 0 71 257.4 3 1067 32 | Processing: 23 100 29.4 99 1104 33 | Waiting: 13 49 23.0 43 1042 34 | Total: 23 171 247.3 105 2026 35 | 36 | Percentage of the requests served within a certain time (ms) 37 | 50% 105 38 | 66% 114 39 | 75% 122 40 | 80% 129 41 | 90% 159 42 | 95% 1084 43 | 98% 1093 44 | 99% 1104 45 | 100% 2026 (longest request) 46 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_10k-rows-query.txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20system.10k_rows 13 | Document Length: 575631 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 854.886 seconds 17 | Complete requests: 100000 18 | Failed requests: 0 19 | Write errors: 0 20 | Total transferred: 57605300000 bytes 21 | HTML transferred: 57563100000 bytes 22 | Requests per second: 116.97 [#/sec] (mean) 23 | Time per request: 170.977 [ms] (mean) 24 | Time per request: 8.549 [ms] (mean, across all concurrent requests) 25 | Transfer rate: 65804.27 [Kbytes/sec] received 26 | 27 | Connection Times (ms) 28 | min mean[+/-sd] median max 29 | Connect: 0 80 267.8 5 1068 30 | Processing: 12 91 15.5 92 187 31 | Waiting: 4 13 5.4 12 122 32 | Total: 12 171 257.3 99 1208 33 | 34 | Percentage of the requests served within a certain time (ms) 35 | 50% 99 36 | 66% 102 37 | 75% 105 38 | 80% 107 39 | 90% 119 40 | 95% 1082 41 | 98% 1094 42 | 99% 1102 43 | 100% 1208 (longest request) 44 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_constant-query(jdbc).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20jdbc%28%27mariadb%27%2C%27constant%27%29 13 | Document Length: 2 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 925.087 seconds 17 | Complete requests: 100000 18 | Failed requests: 5813 19 | (Connect: 0, Receive: 0, Length: 5813, Exceptions: 0) 20 | Write errors: 0 21 | Non-2xx responses: 5813 22 | Total transferred: 42683455 bytes 23 | HTML transferred: 982740 bytes 24 | Requests per second: 108.10 [#/sec] (mean) 25 | Time per request: 185.017 [ms] (mean) 26 | Time per request: 9.251 [ms] (mean, across all concurrent requests) 27 | Transfer rate: 45.06 [Kbytes/sec] received 28 | 29 | Connection Times (ms) 30 | min mean[+/-sd] median max 31 | Connect: 0 52 225.6 1 1066 32 | Processing: 14 132 235.3 72 3093 33 | Waiting: 13 132 235.4 72 3090 34 | Total: 14 185 327.7 75 4091 35 | 36 | Percentage of the requests served within a certain time (ms) 37 | 50% 75 38 | 66% 82 39 | 75% 88 40 | 80% 96 41 | 90% 1006 42 | 95% 1070 43 | 98% 1099 44 | 99% 1147 45 | 100% 4091 (longest request) 46 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_constant-query(mysql).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20mysql%28%27mariadb%3A3306%27%2C%20%27test%27%2C%20%27constant%27%2C%20%27root%27%2C%20%27root%27%29 13 | Document Length: 2 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 1598.426 seconds 17 | Complete requests: 100000 18 | Failed requests: 0 19 | Write errors: 0 20 | Total transferred: 42400000 bytes 21 | HTML transferred: 200000 bytes 22 | Requests per second: 62.56 [#/sec] (mean) 23 | Time per request: 319.685 [ms] (mean) 24 | Time per request: 15.984 [ms] (mean, across all concurrent requests) 25 | Transfer rate: 25.90 [Kbytes/sec] received 26 | 27 | Connection Times (ms) 28 | min mean[+/-sd] median max 29 | Connect: 0 1 5.7 1 1034 30 | Processing: 7 319 530.1 17 2048 31 | Waiting: 6 318 530.0 16 2047 32 | Total: 7 320 530.1 18 2049 33 | 34 | Percentage of the requests served within a certain time (ms) 35 | 50% 18 36 | 66% 23 37 | 75% 1017 38 | 80% 1018 39 | 90% 1020 40 | 95% 1022 41 | 98% 2021 42 | 99% 2022 43 | 100% 2049 (longest request) 44 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_constant-query(remote).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20remote%28%27ch-server%3A9000%27%2C%20system.constant%2C%20%27default%27%2C%20%27%27%29 13 | Document Length: 2 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 802.212 seconds 17 | Complete requests: 100000 18 | Failed requests: 0 19 | Write errors: 0 20 | Total transferred: 42400000 bytes 21 | HTML transferred: 200000 bytes 22 | Requests per second: 124.66 [#/sec] (mean) 23 | Time per request: 160.442 [ms] (mean) 24 | Time per request: 8.022 [ms] (mean, across all concurrent requests) 25 | Transfer rate: 51.62 [Kbytes/sec] received 26 | 27 | Connection Times (ms) 28 | min mean[+/-sd] median max 29 | Connect: 0 153 362.4 1 3064 30 | Processing: 2 7 3.2 7 65 31 | Waiting: 2 6 3.0 6 64 32 | Total: 2 160 363.2 8 3073 33 | 34 | Percentage of the requests served within a certain time (ms) 35 | 50% 8 36 | 66% 10 37 | 75% 11 38 | 80% 13 39 | 90% 1023 40 | 95% 1025 41 | 98% 1027 42 | 99% 1028 43 | 100% 3073 (longest request) 44 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_constant-query(url).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20url%28%27http%3A%2F%2Fch-server%3A8123%2F%3Fquery%3Dselect%25201%27%2C%20CSV%2C%20%27results%20String%27%29 13 | Document Length: 2 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 801.686 seconds 17 | Complete requests: 100000 18 | Failed requests: 0 19 | Write errors: 0 20 | Total transferred: 42400000 bytes 21 | HTML transferred: 200000 bytes 22 | Requests per second: 124.74 [#/sec] (mean) 23 | Time per request: 160.337 [ms] (mean) 24 | Time per request: 8.017 [ms] (mean, across all concurrent requests) 25 | Transfer rate: 51.65 [Kbytes/sec] received 26 | 27 | Connection Times (ms) 28 | min mean[+/-sd] median max 29 | Connect: 0 150 358.8 1 1067 30 | Processing: 3 11 4.2 10 98 31 | Waiting: 3 9 3.9 9 95 32 | Total: 3 160 359.4 11 1123 33 | 34 | Percentage of the requests served within a certain time (ms) 35 | 50% 11 36 | 66% 13 37 | 75% 15 38 | 80% 17 39 | 90% 1023 40 | 95% 1025 41 | 98% 1027 42 | 99% 1028 43 | 100% 1123 (longest request) 44 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_constant-query.txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%1 13 | Document Length: 182 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 797.775 seconds 17 | Complete requests: 100000 18 | Failed requests: 0 19 | Write errors: 0 20 | Non-2xx responses: 100000 21 | Total transferred: 39700000 bytes 22 | HTML transferred: 18200000 bytes 23 | Requests per second: 125.35 [#/sec] (mean) 24 | Time per request: 159.555 [ms] (mean) 25 | Time per request: 7.978 [ms] (mean, across all concurrent requests) 26 | Transfer rate: 48.60 [Kbytes/sec] received 27 | 28 | Connection Times (ms) 29 | min mean[+/-sd] median max 30 | Connect: 0 156 365.8 1 1062 31 | Processing: 1 3 1.8 3 48 32 | Waiting: 1 3 1.7 2 48 33 | Total: 1 159 366.7 4 1077 34 | 35 | Percentage of the requests served within a certain time (ms) 36 | 50% 4 37 | 66% 5 38 | 75% 6 39 | 80% 7 40 | 90% 1023 41 | 95% 1025 42 | 98% 1027 43 | 99% 1029 44 | 100% 1077 (longest request) 45 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_ping.txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /ping 13 | Document Length: 4 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 801.367 seconds 17 | Complete requests: 100000 18 | Failed requests: 0 19 | Write errors: 0 20 | Total transferred: 24100000 bytes 21 | HTML transferred: 400000 bytes 22 | Requests per second: 124.79 [#/sec] (mean) 23 | Time per request: 160.273 [ms] (mean) 24 | Time per request: 8.014 [ms] (mean, across all concurrent requests) 25 | Transfer rate: 29.37 [Kbytes/sec] received 26 | 27 | Connection Times (ms) 28 | min mean[+/-sd] median max 29 | Connect: 0 158 367.3 2 1066 30 | Processing: 0 2 1.2 2 58 31 | Waiting: 0 2 1.2 2 57 32 | Total: 1 160 367.5 4 1075 33 | 34 | Percentage of the requests served within a certain time (ms) 35 | 50% 4 36 | 66% 5 37 | 75% 6 38 | 80% 7 39 | 90% 1024 40 | 95% 1024 41 | 98% 1025 42 | 99% 1026 43 | 100% 1075 (longest request) 44 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_url(clickhouse).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20url%28%27http%3A%2F%2Fch-server%3A8123%2Fping%27%2C%20CSV%2C%20%27result%20String%27%29 13 | Document Length: 4 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 801.448 seconds 17 | Complete requests: 100000 18 | Failed requests: 3 19 | (Connect: 0, Receive: 0, Length: 3, Exceptions: 0) 20 | Write errors: 0 21 | Non-2xx responses: 3 22 | Total transferred: 42600159 bytes 23 | HTML transferred: 400414 bytes 24 | Requests per second: 124.77 [#/sec] (mean) 25 | Time per request: 160.290 [ms] (mean) 26 | Time per request: 8.014 [ms] (mean, across all concurrent requests) 27 | Transfer rate: 51.91 [Kbytes/sec] received 28 | 29 | Connection Times (ms) 30 | min mean[+/-sd] median max 31 | Connect: 0 152 361.4 1 1067 32 | Processing: 2 8 6.4 7 1016 33 | Waiting: 2 7 6.2 6 1016 34 | Total: 3 160 362.3 8 1077 35 | 36 | Percentage of the requests served within a certain time (ms) 37 | 50% 8 38 | 66% 10 39 | 75% 12 40 | 80% 14 41 | 90% 1023 42 | 95% 1025 43 | 98% 1026 44 | 99% 1028 45 | 100% 1077 (longest request) 46 | -------------------------------------------------------------------------------- /misc/perf-test/results/clickhouse_url(jdbc-bridge).txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking ch-server (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: ch-server 10 | Server Port: 8123 11 | 12 | Document Path: /?query=select%20%2A%20from%20url%28%27http%3A%2F%2Fjdbc-bridge%3A9019%2Fping%27%2C%20CSV%2C%20%27result%20String%27%29 13 | Document Length: 4 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 811.299 seconds 17 | Complete requests: 100000 18 | Failed requests: 446 19 | (Connect: 0, Receive: 0, Length: 446, Exceptions: 0) 20 | Write errors: 0 21 | Non-2xx responses: 446 22 | Total transferred: 42623638 bytes 23 | HTML transferred: 461548 bytes 24 | Requests per second: 123.26 [#/sec] (mean) 25 | Time per request: 162.260 [ms] (mean) 26 | Time per request: 8.113 [ms] (mean, across all concurrent requests) 27 | Transfer rate: 51.31 [Kbytes/sec] received 28 | 29 | Connection Times (ms) 30 | min mean[+/-sd] median max 31 | Connect: 0 147 356.4 1 3053 32 | Processing: 3 15 69.7 9 1032 33 | Waiting: 2 14 69.8 8 1032 34 | Total: 3 162 375.4 10 3066 35 | 36 | Percentage of the requests served within a certain time (ms) 37 | 50% 10 38 | 66% 12 39 | 75% 14 40 | 80% 15 41 | 90% 1022 42 | 95% 1026 43 | 98% 1029 44 | 99% 1034 45 | 100% 3066 (longest request) 46 | -------------------------------------------------------------------------------- /misc/perf-test/results/jdbc-bridge_ping.txt: -------------------------------------------------------------------------------- 1 | This is ApacheBench, Version 2.3 <$Revision: 1430300 $> 2 | Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ 3 | Licensed to The Apache Software Foundation, http://www.apache.org/ 4 | 5 | Benchmarking jdbc-bridge (be patient) 6 | 7 | 8 | Server Software: 9 | Server Hostname: jdbc-bridge 10 | Server Port: 9019 11 | 12 | Document Path: /ping 13 | Document Length: 4 bytes 14 | 15 | Concurrency Level: 20 16 | Time taken for tests: 804.017 seconds 17 | Complete requests: 100000 18 | Failed requests: 0 19 | Write errors: 0 20 | Total transferred: 4200000 bytes 21 | HTML transferred: 400000 bytes 22 | Requests per second: 124.38 [#/sec] (mean) 23 | Time per request: 160.803 [ms] (mean) 24 | Time per request: 8.040 [ms] (mean, across all concurrent requests) 25 | Transfer rate: 5.10 [Kbytes/sec] received 26 | 27 | Connection Times (ms) 28 | min mean[+/-sd] median max 29 | Connect: 0 151 361.4 0 3063 30 | Processing: 1 9 2.8 9 48 31 | Waiting: 1 9 2.8 9 48 32 | Total: 1 161 361.1 10 3066 33 | 34 | Percentage of the requests served within a certain time (ms) 35 | 50% 10 36 | 66% 11 37 | 75% 11 38 | 80% 13 39 | 90% 1021 40 | 95% 1027 41 | 98% 1032 42 | 99% 1035 43 | 100% 3066 (longest request) 44 | -------------------------------------------------------------------------------- /misc/quick-start/ch-server/config/jdbc_bridge_config.xml: -------------------------------------------------------------------------------- 1 | 2 | 0.0.0.0 3 | 4 | 5 | jdbc-bridge 6 | 9019 7 | 8 | -------------------------------------------------------------------------------- /misc/quick-start/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | # Only 'ch-server' and 'jdbc-bridge' are mandatory. 4 | # You may remove any db-xxx to save memory. 5 | services: 6 | db-elasticsearch: 7 | image: amazon/opendistro-for-elasticsearch:1.13.2 8 | hostname: db-elasticsearch 9 | environment: 10 | # admin/admin 11 | # curl -XGET https://localhost:9200 -u admin:admin --insecure 12 | # curl -XGET https://localhost:9200/_cat/nodes?v -u admin:admin --insecure 13 | # curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure 14 | discovery.type: single-node 15 | mem_limit: 768m 16 | restart: always 17 | 18 | db-mariadb10: 19 | image: mariadb:10 20 | hostname: db-mariadb10 21 | environment: 22 | MYSQL_DATABASE: test 23 | MYSQL_ROOT_PASSWORD: root 24 | mem_limit: 256m 25 | restart: always 26 | 27 | db-postgres13: 28 | image: postgres:13 29 | hostname: db-postgres13 30 | environment: 31 | POSTGRES_DB: test 32 | POSTGRES_USER: sa 33 | POSTGRES_PASSWORD: sa 34 | mem_limit: 256m 35 | restart: always 36 | 37 | db-mysql5: 38 | image: mysql:5 39 | hostname: db-mysql5 40 | environment: 41 | MYSQL_DATABASE: test 42 | MYSQL_ROOT_PASSWORD: root 43 | mem_limit: 256m 44 | restart: always 45 | 46 | db-mysql8: 47 | image: mysql:8 48 | hostname: db-mysql8 49 | environment: 50 | MYSQL_DATABASE: test 51 | MYSQL_ROOT_PASSWORD: root 52 | mem_limit: 256m 53 | restart: always 54 | 55 | ch-server: 56 | image: clickhouse/clickhouse-server:22.3 57 | hostname: ch-server 58 | volumes: 59 | - ./ch-server/config:/etc/clickhouse-server/config.d 60 | #- ./ch-server/data:/var/lib/clickhouse 61 | #- ./ch-server/logs:/var/log/clickhouse 62 | mem_limit: 256m 63 | restart: always 64 | 65 | jdbc-bridge: 66 | image: clickhouse/jdbc-bridge:2.1 67 | hostname: jdbc-bridge 68 | # In general you don't need to define any environment variable 69 | # Below are all default settings just for demonstration 70 | environment: 71 | CONFIG_DIR: config # configuration directory 72 | SERIAL_MODE: "false" # whether run query in serial mode or not 73 | HTTPD_CONFIG_FILE: httpd.json # httpd configuration file 74 | SERVER_CONFIG_FILE: server.json # server configuration file 75 | VERTX_CONFIG_FILE: vertx.json # vertx configuration file 76 | DATASOURCE_CONFIG_DIR: datasources # named datasource directory 77 | DRIVER_DIR: drivers # driver directory 78 | EXTENSION_DIR: extensions # extension directory 79 | QUERY_CONFIG_DIR: queries # named query directory 80 | CUSTOM_DRIVER_LOADER: "true" # whether use custom driver loader or not 81 | JDBC_BRIDGE_JVM_OPTS: # use CPU and memory allocated by container 82 | 83 | # You may want to keep datasources, queries, SQL scripts, and maybe drivers in a git repo 84 | volumes: 85 | - ./jdbc-bridge/config:/app/config 86 | - ./jdbc-bridge/drivers:/app/drivers 87 | - ./jdbc-bridge/scripts:/app/scripts 88 | mem_limit: 256m 89 | restart: always 90 | -------------------------------------------------------------------------------- /misc/quick-start/jdbc-bridge/config/datasources/ch-server.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/datasource.jschema", 3 | "ch-server": { 4 | "aliases": ["self"], 5 | "driverClassName": "com.clickhouse.ClickHouseDriver", 6 | "jdbcUrl": "jdbc:clickhouse://ch-server:8123/system", 7 | "username": "default", 8 | "password": "", 9 | "initializationFailTimeout": 0, 10 | "minimumIdle": 0, 11 | "maximumPoolSize": 10 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /misc/quick-start/jdbc-bridge/config/datasources/elasticsearch.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/datasource.jschema", 3 | "elasticsearch": { 4 | "driverUrls": [ 5 | "https://repo1.maven.org/maven2/com/amazon/opendistroforelasticsearch/client/opendistro-sql-jdbc/1.13.0.0/opendistro-sql-jdbc-1.13.0.0.jar" 6 | ], 7 | "readOnly": true, 8 | "connectionTestQuery": "", 9 | "driverClassName": "com.amazon.opendistroforelasticsearch.jdbc.Driver", 10 | "jdbcUrl": "jdbc:elasticsearch://elasticsearch:9200?useSSL=true&trustSelfSigned=true&hostnameVerification=false", 11 | "username": "admin", 12 | "password": "admin", 13 | "initializationFailTimeout": 0, 14 | "minimumIdle": 0, 15 | "maximumPoolSize": 10 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /misc/quick-start/jdbc-bridge/config/datasources/mariadb10.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/datasource.jschema", 3 | "mariadb10": { 4 | "driverUrls": [ 5 | "https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.7.4/mariadb-java-client-2.7.4.jar" 6 | ], 7 | "driverClassName": "org.mariadb.jdbc.Driver", 8 | "jdbcUrl": "jdbc:mariadb://db-mariadb10/test?useSSL=false&useCompression=false&useOldAliasMetadataBehavior=true&allowMultiQueries=true", 9 | "dataSource": { 10 | "user": "root", 11 | "password": "root" 12 | }, 13 | "initializationFailTimeout": 0, 14 | "minimumIdle": 0, 15 | "maximumPoolSize": 10 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /misc/quick-start/jdbc-bridge/config/datasources/mysql5.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/datasource.jschema", 3 | "mysql5": { 4 | "driverUrls": [ 5 | "https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar" 6 | ], 7 | "driverClassName": "com.mysql.jdbc.Driver", 8 | "jdbcUrl": "jdbc:mysql://db-mysql5/test?useSSL=false&rewriteBatchedStatements=true&useCompression=false&useOldAliasMetadataBehavior=true&allowMultiQueries=true", 9 | "username": "root", 10 | "password": "root", 11 | "initializationFailTimeout": 0, 12 | "minimumIdle": 0, 13 | "maximumPoolSize": 10 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /misc/quick-start/jdbc-bridge/config/datasources/mysql8.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/datasource.jschema", 3 | "mysql8": { 4 | "driverUrls": [ 5 | "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.26/mysql-connector-java-8.0.26.jar" 6 | ], 7 | "driverClassName": "com.mysql.cj.jdbc.Driver", 8 | "jdbcUrl": "jdbc:mysql://db-mysql8/test?allowPublicKeyRetrieval=true&useSSL=false&useCompression=false&useOldAliasMetadataBehavior=true&allowMultiQueries=true", 9 | "username": "root", 10 | "password": "root", 11 | "initializationFailTimeout": 0, 12 | "minimumIdle": 0, 13 | "maximumPoolSize": 10 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /misc/quick-start/jdbc-bridge/config/datasources/postgres13.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/datasource.jschema", 3 | "postgres13": { 4 | "converter": { 5 | "mappings": [{ "nativeType": "bool", "toType": "String" }] 6 | }, 7 | "driverUrls": [ 8 | "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.24/postgresql-42.2.24.jar" 9 | ], 10 | "driverClassName": "org.postgresql.Driver", 11 | "jdbcUrl": "jdbc:postgresql://db-postgres13/test", 12 | "username": "sa", 13 | "password": "sa", 14 | "initializationFailTimeout": 0, 15 | "minimumIdle": 0, 16 | "maximumPoolSize": 10 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /misc/quick-start/jdbc-bridge/config/queries/show-query-logs.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/datasource.jschema", 3 | "show-query-logs": { 4 | "query": "scripts/show-query-logs.sql", 5 | "columns": [ 6 | { 7 | "name": "query_id", 8 | "type": "String", 9 | "nullable": false 10 | }, 11 | { 12 | "name": "type", 13 | "type": "String", 14 | "nullable": false 15 | }, 16 | { 17 | "name": "event_time", 18 | "type": "DateTime", 19 | "nullable": false 20 | }, 21 | { 22 | "name": "query_start_time", 23 | "type": "DateTime", 24 | "nullable": false 25 | }, 26 | { 27 | "name": "query", 28 | "type": "String", 29 | "nullable": true 30 | }, 31 | { 32 | "name": "user", 33 | "type": "String", 34 | "nullable": true 35 | } 36 | ] 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /misc/quick-start/jdbc-bridge/config/schemas/query-log.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../../docker/config/schema.jschema", 3 | "query-log": { 4 | "columns": [ 5 | { 6 | "name": "query_id", 7 | "type": "String", 8 | "nullable": false 9 | }, 10 | { 11 | "name": "type", 12 | "type": "String", 13 | "nullable": false 14 | }, 15 | { 16 | "name": "event_time", 17 | "type": "DateTime", 18 | "nullable": false 19 | }, 20 | { 21 | "name": "query_start_time", 22 | "type": "DateTime", 23 | "nullable": false 24 | }, 25 | { 26 | "name": "query", 27 | "type": "String", 28 | "nullable": true 29 | }, 30 | { 31 | "name": "user", 32 | "type": "String", 33 | "nullable": true 34 | } 35 | ] 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /misc/quick-start/jdbc-bridge/scripts/show-query-logs.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from system.query_logs 3 | -------------------------------------------------------------------------------- /src/changelog: -------------------------------------------------------------------------------- 1 | * Sun Oct 18 2020 Zhichun Wu 2 | - refactor to make it more extensible along with new features like ScriptDataSource 3 | * Sun Jan 19 2020 Zhichun Wu 4 | - re-write JDBC bridge for more features 5 | -------------------------------------------------------------------------------- /src/main/bin/clickhouse-jdbc-bridge: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Copyright 2019-2021, Zhichun Wu 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | set -e 20 | 21 | [ "$TRACE" ] && set -x 22 | 23 | BASEDIR=$(dirname "$0") 24 | INIT_SCRIPT="/etc/clickhouse-jdbc-bridge/set-env.sh" 25 | JAVA_CMD="java" 26 | 27 | init_env() { 28 | if [ -f $INIT_SCRIPT ]; then 29 | echo "Executing [$INIT_SCRIPT] to set environment variables..." 30 | . $INIT_SCRIPT 31 | else 32 | echo "[$INIT_SCRIPT] is not found, will use default environment variables" 33 | fi 34 | 35 | # check if we have JRE 36 | if [ -n "$JAVA_HOME" ] && [ -x "$JAVA_HOME/bin/java" ]; then 37 | JAVA_CMD="$JAVA_HOME/bin/java" 38 | elif ! type -p java > /dev/null; then 39 | echo "ERROR: Java needs to be installed first" 40 | exit 1 41 | fi 42 | } 43 | 44 | start_server() { 45 | # override below environment variables as needed in $INIT_SCRIPT 46 | : ${APP_PACKAGE:="/usr/local/lib/java/clickhouse-jdbc-bridge-shaded.jar"} 47 | : ${JVM_ARGS:="-Xmx512m"} 48 | : ${WORK_DIRECTORY:=$(dirname "$INIT_SCRIPT")} 49 | 50 | # kill server if it's running 51 | kill -9 $(ps -ef | grep java | grep "$APP_PACKAGE" | awk '{print $2}') 2>/dev/null || true 52 | 53 | if [ ! -f "$APP_PACKAGE" ]; then 54 | echo "ERROR: ClickHouse JDBC Bridge could not start because [$APP_PACKAGE] is missing" 55 | exit 1 56 | fi 57 | 58 | if [ ! -d "$WORK_DIRECTORY" ]; then 59 | echo "WARN: Work directory [$WORK_DIRECTORY] does not exist" 60 | WORK_DIRECTORY=$(dirname "$INIT_SCRIPT") 61 | fi 62 | 63 | echo "Work directory is set to [$WORK_DIRECTORY]" 64 | cd "$WORK_DIRECTORY" 65 | java $JVM_ARGS -Djava.util.logging.config.file=$WORK_DIRECTORY/logging.properties \ 66 | -Duser.dir="$WORK_DIRECTORY" -jar "$APP_PACKAGE" 67 | } 68 | 69 | init_env 70 | 71 | start_server 72 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/DataAccessException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.util.Objects; 19 | 20 | /** 21 | * Thrown by {@link DataTableReader} when a problem occurs. 22 | * 23 | * @since 2.0 24 | */ 25 | public class DataAccessException extends RuntimeException { 26 | private static final long serialVersionUID = -37476939631473275L; 27 | 28 | static final String ERROR_BEGIN = "Failed to access ["; 29 | static final String ERROR_END = "] due to: "; 30 | 31 | static final String UNKNOWN_ERROR = "unknown error"; 32 | 33 | static String buildErrorMessage(String dataSourceId, String message, Throwable cause) { 34 | return new StringBuilder().append(ERROR_BEGIN).append(dataSourceId).append(ERROR_END).append( 35 | message != null && !message.isEmpty() ? message : (cause == null ? UNKNOWN_ERROR : cause.getMessage())) 36 | .toString(); 37 | } 38 | 39 | /** 40 | * Constructor for {@code DataAccessException}. 41 | * 42 | * @param dataSourceId datasource id 43 | * @param cause root cause 44 | */ 45 | public DataAccessException(String dataSourceId, Throwable cause) { 46 | this(dataSourceId, null, cause); 47 | } 48 | 49 | /** 50 | * Constructor for {@code DataAccessException}. 51 | * 52 | * @param dataSourceId datasource id 53 | * @param message detailed message 54 | * @param cause root cause 55 | */ 56 | public DataAccessException(String dataSourceId, String message, Throwable cause) { 57 | super(buildErrorMessage(Objects.requireNonNull(dataSourceId), message, cause), Objects.requireNonNull(cause)); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/DataSourceStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.util.Date; 19 | import java.util.Objects; 20 | 21 | /** 22 | * This class defines run-time statistics of a datasource. 23 | * 24 | * @since 2.0 25 | */ 26 | public class DataSourceStats implements UsageStats { 27 | private final String idOrAlias; 28 | private final int instance; 29 | private final boolean alias; 30 | private final Date createDateTime; 31 | private final String type; 32 | 33 | private final String defaults; 34 | private final String parameters; 35 | private final String customColumns; 36 | private final String cacheUsage; 37 | private final String poolUsage; 38 | 39 | public DataSourceStats(String id, NamedDataSource ds) { 40 | this.idOrAlias = Objects.requireNonNull(id); 41 | this.instance = ds.hashCode(); 42 | this.alias = !this.idOrAlias.equals(Objects.requireNonNull(ds).getId()); 43 | this.createDateTime = ds.getCreateDateTime(); 44 | this.type = ds.getType(); 45 | 46 | this.defaults = ds.getDefaultValuesAsJsonString(); 47 | this.parameters = ds.getParametersAsJsonString(); 48 | this.customColumns = ds.getCustomColumnsAsJsonString(); 49 | this.cacheUsage = ds.getCacheUsage(); 50 | this.poolUsage = ds.getPoolUsage(); 51 | } 52 | 53 | public String getName() { 54 | return this.idOrAlias; 55 | } 56 | 57 | public int getInstance() { 58 | return this.instance; 59 | } 60 | 61 | public boolean isAlias() { 62 | return this.alias; 63 | } 64 | 65 | public Date getCreateDateTime() { 66 | return this.createDateTime; 67 | } 68 | 69 | public String getType() { 70 | return this.type; 71 | } 72 | 73 | public String getParameters() { 74 | return this.parameters; 75 | } 76 | 77 | public String getCustomColumns() { 78 | return this.customColumns; 79 | } 80 | 81 | public String getDefaults() { 82 | return this.defaults; 83 | } 84 | 85 | public String getCacheUsage() { 86 | return this.cacheUsage; 87 | } 88 | 89 | public String getPoolUsage() { 90 | return this.poolUsage; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/DataType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | // https://clickhouse.tech/docs/en/sql-reference/data-types/ 19 | public enum DataType { 20 | // Boolean 21 | Bool(1, 4, 0), // interpreted as Int8, recommend to use UInt8(0 or 1) instead 22 | 23 | // Signed 24 | Int8(1, 4, 0), Int16(2, 6, 0), Int32(4, 11, 0), Int64(8, 20, 0), Int128(16, 20, 0), Int256(32, 40, 0), 25 | 26 | // Unsigned 27 | UInt8(1, 3, 0), UInt16(2, 5, 0), UInt32(4, 10, 0), UInt64(8, 19, 0), UInt128(16, 20, 0), UInt256(32, 39, 0), 28 | 29 | // Floating point 30 | Float32(4, 8, 8), Float64(16, 17, 17), 31 | 32 | // Date time 33 | Date(4, 10, 0), DateTime(8, 19, 0), DateTime64(16, 38, 18), 34 | 35 | // Decimals 36 | Decimal(32, 76, 76), Decimal32(4, 9, 9), Decimal64(8, 18, 18), Decimal128(16, 38, 38), Decimal256(32, 76, 76), 37 | 38 | // Misc 39 | Enum(1, 4, 0), Enum8(1, 4, 0), Enum16(2, 6, 0), IPv4(4, 10, 0), IPv6(16, 0, 0), FixedStr(0, 0, 0), Str(0, 0, 0), 40 | UUID(16, 20, 0); 41 | 42 | // TODO: support complex types: 43 | // Array, Tuple, Nested, AggregateFunction, SimpleAggregateFunction 44 | 45 | public static final String ALIAS_BOOLEAN = "Boolean"; 46 | public static final String ALIAS_STRING = "String"; 47 | public static final String ALIAS_FIXED_STRING = "FixedString"; 48 | 49 | public static final int DEFAULT_DECIMAL_PRECISON = 10; 50 | public static final int DEFAULT_DECIMAL_SCALE = 4; 51 | public static final int DEFAULT_DECIMAL32_PRECISON = 9; 52 | public static final int DEFAULT_DECIMAL32_SCALE = 2; 53 | public static final int DEFAULT_DECIMAL64_PRECISON = 18; 54 | public static final int DEFAULT_DECIMAL64_SCALE = 4; 55 | public static final int DEFAULT_DECIMAL128_PRECISON = 38; 56 | public static final int DEFAULT_DECIMAL128_SCALE = 8; 57 | public static final int DEFAULT_DECIMAL256_PRECISON = 76; 58 | public static final int DEFAULT_DECIMAL256_SCALE = 16; 59 | 60 | // https://clickhouse.tech/docs/en/sql-reference/data-types/decimal/ 61 | public static final int MAX_PRECISON = 76; 62 | 63 | public static final boolean DEFAULT_NULLABLE = true; 64 | public static final int DEFAULT_LENGTH = 0; 65 | public static final int DEFAULT_PRECISION = 0; 66 | public static final int DEFAULT_SCALE = 0; 67 | 68 | public static final int MAX_DATETIME64_PRECISION = 38; // 19 + 1 + 18 69 | public static final int MAX_DATETIME64_SCALE = 18; 70 | public static final int DEFAULT_DATETIME64_PRECISION = 23; // 19 + 1 + 3 71 | // Tick size (precision): 10-precision seconds 72 | public static final int DEFAULT_DATETIME64_SCALE = 3; 73 | 74 | /** 75 | * Replacement of {@link #valueOf(String)}. 76 | * 77 | * @param value case-insensitive string representation of the data type 78 | * @return data type 79 | */ 80 | public static DataType from(String value) { 81 | DataType t = Str; 82 | 83 | if (value == null || ALIAS_STRING.equalsIgnoreCase(value)) { 84 | t = Str; 85 | } else if (ALIAS_BOOLEAN.equalsIgnoreCase(value)) { 86 | t = Bool; 87 | } else if (ALIAS_FIXED_STRING.equalsIgnoreCase(value)) { 88 | t = FixedStr; 89 | } else { 90 | for (DataType d : DataType.values()) { 91 | if (d.name().equalsIgnoreCase(value)) { 92 | t = d; 93 | break; 94 | } 95 | } 96 | } 97 | 98 | return t; 99 | } 100 | 101 | private final int length; 102 | private final int precision; 103 | private final int scale; 104 | 105 | private DataType(int length, int precision, int scale) { 106 | this.length = length < 0 ? DEFAULT_LENGTH : length; 107 | this.precision = precision < 0 ? DEFAULT_PRECISION : precision; 108 | this.scale = scale < 0 ? DEFAULT_SCALE : (scale > this.precision ? this.precision : scale); 109 | } 110 | 111 | /** 112 | * Get length in byte. 113 | * 114 | * @return length in byte, zero stands for unlimited 115 | */ 116 | public int getLength() { 117 | return length; 118 | } 119 | 120 | public int getPrecision() { 121 | return precision; 122 | } 123 | 124 | public int getScale() { 125 | return scale; 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/DataTypeConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.math.BigDecimal; 19 | import java.math.BigInteger; 20 | import java.sql.JDBCType; 21 | import java.time.LocalDate; 22 | import java.time.LocalDateTime; 23 | import java.time.format.DateTimeFormatter; 24 | import java.util.Date; 25 | 26 | /** 27 | * This interface defines converters to map from one data type to another. 28 | * 29 | * @since 2.0 30 | */ 31 | public interface DataTypeConverter { 32 | static final String M_TYPE_TEXT = "type text"; 33 | static final String M_TYPE_NUMBER = "type number"; 34 | static final String M_TYPE_DATE = "type date"; 35 | static final String M_TYPE_DATETIME = "type datetime"; 36 | 37 | static final String M_FACET_INT8 = "Int8.Type"; 38 | static final String M_FACET_INT16 = "Int16.Type"; 39 | static final String M_FACET_INT32 = "Int32.Type"; 40 | static final String M_FACET_INT64 = "Int64.Type"; 41 | static final String M_FACET_SINGLE = "Single.Type"; 42 | static final String M_FACET_DOUBLE = "Double.Type"; 43 | 44 | @SuppressWarnings("unchecked") 45 | default T as(Class type, Object value) { 46 | final Object result; 47 | 48 | if (Boolean.class.equals(type)) { 49 | if (value instanceof Boolean) { 50 | result = value; 51 | } else if (value instanceof Number) { 52 | result = ((Number) value).intValue() != 0; 53 | } else { 54 | result = Boolean.parseBoolean(String.valueOf(value)); 55 | } 56 | } else if (Byte.class.equals(type)) { 57 | if (value instanceof Boolean) { 58 | result = (boolean) value ? (byte) 0 : (byte) 1; 59 | } else if (value instanceof Number) { 60 | result = ((Number) value).byteValue(); 61 | } else { 62 | result = Byte.parseByte(String.valueOf(value)); 63 | } 64 | } else if (Short.class.equals(type)) { 65 | if (value instanceof Boolean) { 66 | result = (boolean) value ? (short) 0 : (short) 1; 67 | } else if (value instanceof Number) { 68 | result = ((Number) value).shortValue(); 69 | } else { 70 | result = Short.parseShort(String.valueOf(value)); 71 | } 72 | } else if (Integer.class.equals(type)) { 73 | if (value instanceof Boolean) { 74 | result = (boolean) value ? 0 : 1; 75 | } else if (value instanceof Number) { 76 | result = ((Number) value).intValue(); 77 | } else { 78 | result = Integer.parseInt(String.valueOf(value)); 79 | } 80 | } else if (Long.class.equals(type)) { 81 | if (value instanceof Boolean) { 82 | result = (boolean) value ? 0L : 1L; 83 | } else if (value instanceof Number) { 84 | result = ((Number) value).longValue(); 85 | } else { 86 | result = Long.parseLong(String.valueOf(value)); 87 | } 88 | } else if (Float.class.equals(type)) { 89 | if (value instanceof Boolean) { 90 | result = (boolean) value ? 0.0F : 1.0F; 91 | } else if (value instanceof Number) { 92 | result = ((Number) value).floatValue(); 93 | } else { 94 | result = Float.parseFloat(String.valueOf(value)); 95 | } 96 | } else if (Double.class.equals(type)) { 97 | if (value instanceof Boolean) { 98 | result = (boolean) value ? 0.0D : 1.0D; 99 | } else if (value instanceof Number) { 100 | result = ((Number) value).doubleValue(); 101 | } else { 102 | result = Double.parseDouble(String.valueOf(value)); 103 | } 104 | } else if (BigInteger.class.equals(type)) { 105 | if (value instanceof Boolean) { 106 | result = (boolean) value ? BigInteger.ZERO : BigInteger.ONE; 107 | } else if (value instanceof BigInteger) { 108 | result = value; 109 | } else { 110 | result = new BigInteger(String.valueOf(value)); 111 | } 112 | } else if (BigDecimal.class.equals(type)) { 113 | if (value instanceof Boolean) { 114 | result = (boolean) value ? BigDecimal.ZERO : BigDecimal.ONE; 115 | } else if (value instanceof BigDecimal) { 116 | result = value; 117 | } else { 118 | result = new BigDecimal(String.valueOf(value)); 119 | } 120 | } else if (Date.class.equals(type)) { 121 | if (value instanceof Boolean) { 122 | result = new Date((boolean) value ? 0L : 1L); 123 | } else if (value instanceof Number) { 124 | result = new Date(((Number) value).longValue()); 125 | } else if (value instanceof String) { 126 | String str = (String) value; 127 | int len = str.length(); 128 | if (len == 10) { 129 | result = java.sql.Date.valueOf(LocalDate.parse(str, DateTimeFormatter.ISO_LOCAL_DATE)); 130 | } else if (len >= 10 && len <= 16) { 131 | result = java.sql.Date.valueOf(LocalDate.parse(str, DateTimeFormatter.ISO_DATE)); 132 | } else if (len == 19) { 133 | result = java.sql.Timestamp 134 | .valueOf(LocalDateTime.parse(str, DateTimeFormatter.ISO_LOCAL_DATE_TIME)); 135 | } else if (len > 19) { 136 | result = java.sql.Timestamp.valueOf(LocalDateTime.parse(str, DateTimeFormatter.ISO_DATE_TIME)); 137 | } else { 138 | result = java.sql.Date.valueOf(LocalDate.parse(str, DateTimeFormatter.BASIC_ISO_DATE)); 139 | } 140 | } else { 141 | result = value; 142 | } 143 | } else if (String.class.equals(type)) { 144 | result = String.valueOf(value); 145 | } else { 146 | result = value; 147 | } 148 | 149 | return (T) result; 150 | } 151 | 152 | DataType from(JDBCType jdbcType, String typeName, int precision, int scale, boolean signed); 153 | 154 | DataType from(Object javaObject); 155 | 156 | default String toPowerQueryType(DataType type) { 157 | return toMType(type); 158 | } 159 | 160 | default String toMType(DataType type) { 161 | String mType = M_TYPE_TEXT; 162 | 163 | switch (type) { 164 | case Bool: 165 | case Int8: 166 | mType = M_FACET_INT8; 167 | break; 168 | case UInt8: 169 | case Int16: 170 | mType = M_FACET_INT16; 171 | break; 172 | case UInt16: 173 | case Int32: 174 | mType = M_FACET_INT32; 175 | break; 176 | case UInt32: 177 | case Int64: 178 | mType = M_FACET_INT64; 179 | break; 180 | case Float32: 181 | mType = M_FACET_SINGLE; 182 | break; 183 | case Float64: 184 | mType = M_FACET_DOUBLE; 185 | break; 186 | case UInt64: 187 | case Decimal: 188 | case Decimal32: 189 | case Decimal64: 190 | case Decimal128: 191 | case Decimal256: 192 | mType = M_TYPE_NUMBER; 193 | break; 194 | case Date: 195 | mType = M_TYPE_DATE; 196 | break; 197 | case DateTime: 198 | case DateTime64: 199 | mType = M_TYPE_DATETIME; 200 | break; 201 | default: 202 | break; 203 | } 204 | 205 | return mType; 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/DataTypeMapping.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.sql.JDBCType; 19 | 20 | /** 21 | * This defines a mapping from specific JDBC/native type to {@link DataType}. 22 | * Native type is case-sensitive. {@code "*"} represents any type. 23 | * 24 | * @since 2.0 25 | */ 26 | public class DataTypeMapping { 27 | public static final String ANY_NATIVE_TYPE = "*"; 28 | 29 | private final JDBCType fromJdbcType; 30 | private final String fromNativeType; 31 | private final DataType toType; 32 | 33 | private static JDBCType parse(String fromJdbcType) { 34 | JDBCType jdbcType = JDBCType.OTHER; 35 | 36 | if (fromJdbcType != null) { 37 | try { 38 | fromJdbcType = fromJdbcType.trim().toUpperCase(); 39 | jdbcType = JDBCType.valueOf(fromJdbcType); 40 | } catch (RuntimeException e) { 41 | } 42 | } 43 | 44 | return jdbcType; 45 | } 46 | 47 | private static JDBCType parse(int fromJdbcType) { 48 | JDBCType jdbcType = JDBCType.OTHER; 49 | 50 | try { 51 | jdbcType = JDBCType.valueOf(fromJdbcType); 52 | } catch (RuntimeException e) { 53 | } 54 | 55 | return jdbcType; 56 | } 57 | 58 | public DataTypeMapping(String fromJdbcType, String fromNativeType, String toType) { 59 | this(parse(fromJdbcType), fromNativeType, DataType.from(toType)); 60 | } 61 | 62 | public DataTypeMapping(int fromJdbcType, String fromNativeType, DataType toType) { 63 | this(parse(fromJdbcType), fromNativeType, toType); 64 | } 65 | 66 | public DataTypeMapping(JDBCType fromJdbcType, String fromNativeType, DataType toType) { 67 | this.fromJdbcType = fromJdbcType; 68 | this.fromNativeType = ANY_NATIVE_TYPE.equals(fromNativeType) ? ANY_NATIVE_TYPE : fromNativeType; 69 | this.toType = toType; 70 | } 71 | 72 | public JDBCType getSourceJdbcType() { 73 | return fromJdbcType; 74 | } 75 | 76 | public String getSourceNativeType() { 77 | return fromNativeType; 78 | } 79 | 80 | public DataType getMappedType() { 81 | return toType; 82 | } 83 | 84 | public boolean accept(JDBCType jdbcType, String nativeType) { 85 | return fromNativeType != null ? (ANY_NATIVE_TYPE == fromNativeType || fromNativeType.equals(nativeType)) 86 | : fromJdbcType == jdbcType; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/DefaultValues.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import static com.clickhouse.jdbcbridge.core.Utils.EMPTY_STRING; 19 | 20 | import java.math.BigDecimal; 21 | import java.math.BigInteger; 22 | import java.util.Map; 23 | import java.util.TreeMap; 24 | import java.util.Map.Entry; 25 | 26 | import io.vertx.core.json.JsonObject; 27 | 28 | /** 29 | * This class defines default value of all supported ClickHouse data types. 30 | * 31 | * @since 2.0 32 | */ 33 | public class DefaultValues { 34 | public static final String DEFAULT_UUID = "00000000-0000-0000-0000-000000000000"; 35 | 36 | // Boolean 37 | public final TypedParameter Bool; 38 | 39 | // Signed 40 | public final TypedParameter Int8; 41 | public final TypedParameter Int16; 42 | public final TypedParameter Int32; 43 | public final TypedParameter Int64; 44 | public final TypedParameter Int128; 45 | public final TypedParameter Int256; 46 | 47 | // Unsigned 48 | public final TypedParameter UInt8; 49 | public final TypedParameter UInt16; 50 | public final TypedParameter UInt32; 51 | public final TypedParameter UInt64; 52 | public final TypedParameter UInt128; 53 | public final TypedParameter UInt256; 54 | 55 | // Floating point 56 | public final TypedParameter Float32; 57 | public final TypedParameter Float64; 58 | 59 | // Date time 60 | public final TypedParameter Date; 61 | public final TypedParameter Datetime; 62 | public final TypedParameter Datetime64; 63 | 64 | // Decimals 65 | public final TypedParameter Decimal; 66 | public final TypedParameter Decimal32; 67 | public final TypedParameter Decimal64; 68 | public final TypedParameter Decimal128; 69 | public final TypedParameter Decimal256; 70 | 71 | // Misc 72 | public final TypedParameter Enum; 73 | public final TypedParameter Enum8; 74 | public final TypedParameter Enum16; 75 | public final TypedParameter IPv4; // 0.0.0.0 76 | public final TypedParameter IPv6; 77 | public final TypedParameter FixedStr; 78 | public final TypedParameter Str; 79 | public final TypedParameter UUID; 80 | 81 | private final Map> types = new TreeMap<>(); 82 | 83 | public DefaultValues() { 84 | Utils.addTypedParameter(types, this.Bool = new TypedParameter<>(Integer.class, DataType.Bool.name(), 0)); 85 | 86 | Utils.addTypedParameter(types, this.Int8 = new TypedParameter<>(Integer.class, DataType.Int8.name(), 0)); 87 | Utils.addTypedParameter(types, this.Int16 = new TypedParameter<>(Integer.class, DataType.Int16.name(), 0)); 88 | Utils.addTypedParameter(types, this.Int32 = new TypedParameter<>(Integer.class, DataType.Int32.name(), 0)); 89 | Utils.addTypedParameter(types, this.Int64 = new TypedParameter<>(Long.class, DataType.Int64.name(), 0L)); 90 | Utils.addTypedParameter(types, 91 | this.Int128 = new TypedParameter<>(BigInteger.class, DataType.Int128.name(), BigInteger.ZERO)); 92 | Utils.addTypedParameter(types, 93 | this.Int256 = new TypedParameter<>(BigInteger.class, DataType.Int256.name(), BigInteger.ZERO)); 94 | 95 | Utils.addTypedParameter(types, this.UInt8 = new TypedParameter<>(Integer.class, DataType.UInt8.name(), 0)); 96 | Utils.addTypedParameter(types, this.UInt16 = new TypedParameter<>(Integer.class, DataType.UInt16.name(), 0)); 97 | Utils.addTypedParameter(types, this.UInt32 = new TypedParameter<>(Long.class, DataType.UInt32.name(), 0L)); 98 | Utils.addTypedParameter(types, this.UInt64 = new TypedParameter<>(Long.class, DataType.UInt64.name(), 0L)); 99 | Utils.addTypedParameter(types, 100 | this.UInt128 = new TypedParameter<>(BigInteger.class, DataType.UInt128.name(), BigInteger.ZERO)); 101 | Utils.addTypedParameter(types, 102 | this.UInt256 = new TypedParameter<>(BigInteger.class, DataType.UInt256.name(), BigInteger.ZERO)); 103 | 104 | Utils.addTypedParameter(types, this.Float32 = new TypedParameter<>(Float.class, DataType.Float32.name(), 0.0F)); 105 | Utils.addTypedParameter(types, this.Float64 = new TypedParameter<>(Double.class, DataType.Float64.name(), 0.0)); 106 | 107 | Utils.addTypedParameter(types, this.Date = new TypedParameter<>(Integer.class, DataType.Date.name(), 1)); 108 | Utils.addTypedParameter(types, this.Datetime = new TypedParameter<>(Long.class, DataType.DateTime.name(), 1L)); 109 | Utils.addTypedParameter(types, 110 | this.Datetime64 = new TypedParameter<>(Long.class, DataType.DateTime64.name(), 1000L)); 111 | 112 | Utils.addTypedParameter(types, 113 | this.Decimal = new TypedParameter<>(BigDecimal.class, DataType.Decimal.name(), BigDecimal.ZERO)); 114 | Utils.addTypedParameter(types, 115 | this.Decimal32 = new TypedParameter<>(BigDecimal.class, DataType.Decimal32.name(), BigDecimal.ZERO)); 116 | Utils.addTypedParameter(types, 117 | this.Decimal64 = new TypedParameter<>(BigDecimal.class, DataType.Decimal64.name(), BigDecimal.ZERO)); 118 | Utils.addTypedParameter(types, 119 | this.Decimal128 = new TypedParameter<>(BigDecimal.class, DataType.Decimal128.name(), BigDecimal.ZERO)); 120 | Utils.addTypedParameter(types, 121 | this.Decimal256 = new TypedParameter<>(BigDecimal.class, DataType.Decimal256.name(), BigDecimal.ZERO)); 122 | 123 | Utils.addTypedParameter(types, this.Enum = new TypedParameter<>(Integer.class, DataType.Enum.name(), 0)); 124 | Utils.addTypedParameter(types, this.Enum8 = new TypedParameter<>(Integer.class, DataType.Enum8.name(), 0)); 125 | Utils.addTypedParameter(types, this.Enum16 = new TypedParameter<>(Integer.class, DataType.Enum16.name(), 0)); 126 | Utils.addTypedParameter(types, this.IPv4 = new TypedParameter<>(Integer.class, DataType.IPv4.name(), 0)); 127 | Utils.addTypedParameter(types, 128 | this.IPv6 = new TypedParameter<>(String.class, DataType.IPv6.name(), EMPTY_STRING)); 129 | Utils.addTypedParameter(types, 130 | this.FixedStr = new TypedParameter<>(String.class, DataType.FixedStr.name(), EMPTY_STRING)); 131 | Utils.addTypedParameter(types, 132 | this.Str = new TypedParameter<>(String.class, DataType.Str.name(), EMPTY_STRING)); 133 | Utils.addTypedParameter(types, 134 | this.UUID = new TypedParameter<>(String.class, DataType.UUID.name(), DEFAULT_UUID)); 135 | } 136 | 137 | public DefaultValues(JsonObject... params) { 138 | this(); 139 | 140 | for (JsonObject p : params) { 141 | merge(p); 142 | } 143 | } 144 | 145 | public DefaultValues merge(JsonObject p) { 146 | if (p != null) { 147 | for (Entry entry : p) { 148 | String key = entry.getKey(); 149 | String name = DataType.from(key).name(); 150 | TypedParameter tp = this.types.get(name); 151 | if (tp != null) { 152 | tp.merge(p, key); 153 | } 154 | } 155 | } 156 | 157 | return this; 158 | } 159 | 160 | public TypedParameter getTypedValue(DataType type) { 161 | TypedParameter p = this.types.get(type.name()); 162 | if (p == null) { 163 | throw new IllegalArgumentException("unsupported type: " + type.name()); 164 | } 165 | return p; 166 | } 167 | 168 | public String asJsonString() { 169 | JsonObject obj = new JsonObject(); 170 | 171 | for (Map.Entry> t : this.types.entrySet()) { 172 | Object value = t.getValue().getDefaultValue(); 173 | obj.put(t.getKey(), value == null ? null : String.valueOf(value)); 174 | } 175 | 176 | return obj.toString(); 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/DnsResolver.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import com.github.benmanes.caffeine.cache.Cache; 21 | import com.github.benmanes.caffeine.cache.Caffeine; 22 | 23 | import org.xbill.DNS.Lookup; 24 | import org.xbill.DNS.Record; 25 | import org.xbill.DNS.SRVRecord; 26 | import org.xbill.DNS.TextParseException; 27 | import org.xbill.DNS.Type; 28 | 29 | /** 30 | * This class will replace host and port variables in given connection string by 31 | * resolving SRV records. 32 | * 33 | * @since 2.0 34 | */ 35 | public class DnsResolver { 36 | private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(DnsResolver.class); 37 | 38 | private static final String PREFIX_HOST = "host:"; // just host 39 | private static final String PREFIX_PORT = "port:"; // just port 40 | 41 | // TODO a backend thread to reload datasource when its DNS changed 42 | // private final Map dns2ds = new ConcurrentHashMap<>(); 43 | 44 | private final Cache dnsCache = Caffeine.newBuilder().maximumSize(100) 45 | .expireAfterAccess(5, TimeUnit.MINUTES).build(); 46 | 47 | public SRVRecord resolve(String srvDns, boolean basedOnWeight) { 48 | Record[] records = null; 49 | 50 | try { 51 | records = dnsCache.get(srvDns, k -> { 52 | Record[] results = null; 53 | 54 | try { 55 | results = new Lookup(srvDns, Type.SRV).run(); 56 | } catch (TextParseException e) { 57 | } 58 | 59 | return results; 60 | }); 61 | } catch (Exception e) { 62 | log.warn("Not able to resolve given DNS query: [{}]", srvDns); 63 | } 64 | 65 | SRVRecord record = null; 66 | if (records != null) { 67 | if (basedOnWeight) { 68 | for (int i = 0; i < records.length; i++) { 69 | SRVRecord rec = (SRVRecord) records[i]; 70 | if (record == null || record.getWeight() > rec.getWeight()) { 71 | record = rec; 72 | } 73 | } 74 | } else { 75 | record = (SRVRecord) records[0]; 76 | } 77 | } 78 | 79 | return record; 80 | } 81 | 82 | public String apply(String dns) { 83 | boolean onlyHost = false; 84 | boolean onlyPort = false; 85 | 86 | String query = dns; 87 | if (dns.startsWith(PREFIX_HOST)) { 88 | onlyHost = true; 89 | query = dns.substring(PREFIX_HOST.length()); 90 | } else if (dns.startsWith(PREFIX_PORT)) { 91 | onlyPort = true; 92 | query = dns.substring(PREFIX_PORT.length()); 93 | } 94 | 95 | SRVRecord record = resolve(query, false); 96 | if (record != null) { 97 | if (onlyHost) { 98 | dns = record.getName().canonicalize().toString(true); 99 | } else if (onlyPort) { 100 | dns = String.valueOf(record.getPort()); 101 | } else { 102 | dns = new StringBuilder().append(record.getName().canonicalize().toString(true)).append(':') 103 | .append(record.getPort()).toString(); 104 | } 105 | } 106 | 107 | return dns; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/ExpandedUrlClassLoader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.io.File; 19 | import java.net.MalformedURLException; 20 | import java.net.URISyntaxException; 21 | import java.net.URL; 22 | import java.net.URLClassLoader; 23 | import java.nio.file.Files; 24 | import java.nio.file.InvalidPathException; 25 | import java.nio.file.Path; 26 | import java.nio.file.Paths; 27 | import java.util.ArrayList; 28 | import java.util.Arrays; 29 | import java.util.HashSet; 30 | import java.util.List; 31 | import java.util.Objects; 32 | import java.util.Set; 33 | 34 | /** 35 | * Enhanced URL class loader which supports directory. 36 | * 37 | * @since 2.0 38 | */ 39 | public class ExpandedUrlClassLoader extends URLClassLoader { 40 | private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(ExpandedUrlClassLoader.class); 41 | 42 | static final String PROTOCOL_FILE = "file"; 43 | static final String FILE_URL_PREFIX = PROTOCOL_FILE + ":///"; 44 | static final String DRIVER_EXTENSION = ".jar"; 45 | 46 | // not going to use OSGi and maven which are over-complex 47 | protected static URL[] expandURLs(String... urls) { 48 | Set cache = new HashSet<>(); 49 | List list = new ArrayList<>(Objects.requireNonNull(urls).length * 2); 50 | Set negativeSet = new HashSet<>(); 51 | 52 | for (String s : urls) { 53 | if (s == null || s.isEmpty() || cache.contains(s)) { 54 | continue; 55 | } 56 | 57 | boolean isNegative = s.length() > 1 && s.charAt(0) == '!'; 58 | if (isNegative) { 59 | s = s.substring(1); 60 | } 61 | 62 | URL url = null; 63 | try { 64 | url = cache.add(s) ? new URL(s) : null; 65 | } catch (MalformedURLException e) { 66 | // might be a local path? 67 | try { 68 | URL tmp = Paths.get(s).normalize().toFile().toURI().toURL(); 69 | if (cache.add(s = tmp.toString())) { 70 | url = tmp; 71 | } 72 | } catch (InvalidPathException exp) { 73 | log.warn("Skip invalid path [{}]", s); 74 | } catch (MalformedURLException exp) { 75 | log.warn("Skip malformed URL [{}]", s); 76 | } 77 | } 78 | 79 | if (url == null) { 80 | continue; 81 | } 82 | 83 | boolean isValid = true; 84 | if (PROTOCOL_FILE.equals(url.getProtocol())) { 85 | Path path = null; 86 | try { 87 | path = Paths.get(url.toURI()); 88 | } catch (URISyntaxException e) { 89 | isValid = false; 90 | log.warn("Skip invalid URL [{}]", url); 91 | } catch (InvalidPathException e) { 92 | isValid = false; 93 | log.warn("Skip invalid path [{}]", url); 94 | } 95 | 96 | if (path != null && Files.isDirectory(path)) { 97 | File dir = path.normalize().toFile(); 98 | String[] files = dir.list(); 99 | Arrays.sort(files); 100 | for (String file : files) { 101 | if (file.endsWith(DRIVER_EXTENSION)) { 102 | file = new StringBuilder().append(FILE_URL_PREFIX).append(dir.getPath()) 103 | .append(File.separatorChar).append(file).toString(); 104 | 105 | if (isNegative) { 106 | try { 107 | negativeSet.add(new URL(file)); 108 | } catch (Exception e) { 109 | // ignore 110 | } 111 | } else if (cache.add(file)) { 112 | try { 113 | list.add(new URL(file)); 114 | } catch (MalformedURLException e) { 115 | log.warn("Skip invalid file [{}]", file); 116 | } 117 | } else { 118 | log.warn("Discard duplicated file [{}]", file); 119 | } 120 | } 121 | } 122 | } 123 | } 124 | 125 | if (isValid) { 126 | (isNegative ? negativeSet : list).add(url); 127 | } 128 | } 129 | 130 | if (list.removeAll(negativeSet)) { 131 | if (log.isDebugEnabled()) { 132 | log.debug("Excluded URLs: {}", negativeSet); 133 | } 134 | } 135 | 136 | return list.toArray(new URL[list.size()]); 137 | } 138 | 139 | public ExpandedUrlClassLoader(ClassLoader parent, String... urls) { 140 | super(expandURLs(urls), parent == null ? ExpandedUrlClassLoader.class.getClassLoader() : parent); 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/ExtensionManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.util.Map; 19 | import java.util.function.Consumer; 20 | 21 | import io.vertx.core.json.JsonObject; 22 | 23 | /** 24 | * This interface defines extension manager, which manages can be used get/set 25 | * managers for datasource, query and schema, as well as how to reload 26 | * configuration files on disk after change. 27 | * 28 | * @since 2.0 29 | */ 30 | public interface ExtensionManager { 31 | /** 32 | * Get extension implemented by given class. 33 | * 34 | * @param type of the extension 35 | * @param clazz implementation class of the extension 36 | * @return desired extension 37 | */ 38 | Extension getExtension(Class clazz); 39 | 40 | /** 41 | * Get repository manager. 42 | * 43 | * @return repository manager 44 | */ 45 | RepositoryManager getRepositoryManager(); 46 | 47 | /** 48 | * Register a consumer to load configuration files(in JSON format) based on 49 | * given path, regardless it's a directory or a file. And it will be called 50 | * again later for reloading, when there's change detected on disk. 51 | * 52 | * @param configPath path to monitor, in general a relative path under 53 | * configuration path 54 | * @param consumer consumer to handle loaded configuration, regardless it's 55 | * new or changed 56 | */ 57 | void registerConfigLoader(String configPath, Consumer consumer); 58 | 59 | /** 60 | * Get list of named scriptable objects. 61 | * 62 | * @return named scriptable objects 63 | */ 64 | Map getScriptableObjects(); 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/ManagedEntity.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.util.Collections; 19 | import java.util.Date; 20 | import java.util.LinkedHashSet; 21 | import java.util.Objects; 22 | import java.util.Set; 23 | 24 | import io.vertx.core.json.JsonArray; 25 | import io.vertx.core.json.JsonObject; 26 | 27 | /** 28 | * This class defines a entity which can be initialized using JSON format 29 | * configuration. 30 | * 31 | * @since 2.0 32 | */ 33 | public abstract class ManagedEntity { 34 | protected static final String CONF_ID = "id"; 35 | protected static final String CONF_ALIASES = "aliases"; 36 | protected static final String CONF_TYPE = "type"; 37 | 38 | protected final Set aliases; 39 | protected final Date createDateTime; 40 | protected final String digest; 41 | protected final String id; 42 | protected final String type; 43 | 44 | /** 45 | * Constructor of configurable entity. 46 | * 47 | * @param id id of the entity 48 | * @param config configuration in JSON format, {@code id}, {@code type} and 49 | * {@code aliases} properties are reserved for instantiation 50 | */ 51 | protected ManagedEntity(String id, JsonObject config) { 52 | this.aliases = new LinkedHashSet<>(); 53 | this.createDateTime = new Date(); 54 | this.digest = Utils.digest(config); 55 | this.id = id == null && config != null ? config.getString(CONF_ID) : id; 56 | 57 | String defaultType = getClass().getSimpleName(); 58 | if (config != null) { 59 | this.type = config.getString(CONF_TYPE, defaultType); 60 | JsonArray array = config.getJsonArray(CONF_ALIASES); 61 | if (array != null) { 62 | for (Object item : array) { 63 | if ((item instanceof String) && !Utils.EMPTY_STRING.equals(item)) { 64 | this.aliases.add((String) item); 65 | } 66 | } 67 | 68 | this.aliases.remove(id); 69 | } 70 | } else { 71 | this.type = defaultType; 72 | } 73 | } 74 | 75 | /** 76 | * Validate this instance and throw runtime exception if something wrong. 77 | */ 78 | public void validate() { 79 | } 80 | 81 | /** 82 | * Get id of the entity. 83 | * 84 | * @return id of the entity 85 | */ 86 | public final String getId() { 87 | return id; 88 | } 89 | 90 | /** 91 | * Get list of aliases of the entity. 92 | * 93 | * @return aliases of the entity 94 | */ 95 | public final Set getAliases() { 96 | return Collections.unmodifiableSet(this.aliases); 97 | } 98 | 99 | /** 100 | * Get creation datetime of the entity. 101 | * 102 | * @return creation datetime of the entity 103 | */ 104 | public final Date getCreationDateTime() { 105 | return this.createDateTime; 106 | } 107 | 108 | /** 109 | * Get type of the entity. 110 | * 111 | * @return type of the entity 112 | */ 113 | public String getType() { 114 | return Objects.requireNonNull(type); 115 | } 116 | 117 | /** 118 | * Check if given configuration is different from current or not. 119 | * 120 | * @param config configuration in JSON format 121 | * @return true if the given configuration is different from current; false 122 | * otherwise 123 | */ 124 | public final boolean isDifferentFrom(JsonObject config) { 125 | return this.digest == null || this.digest.isEmpty() || !this.digest.equals(Utils.digest(config)); 126 | } 127 | 128 | public abstract UsageStats getUsage(String idOrAlias); 129 | } 130 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/NamedQuery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.util.Objects; 19 | 20 | import io.vertx.core.json.JsonObject; 21 | 22 | /** 23 | * This class defines a named query, which is composed of query, schema and 24 | * parameters. 25 | * 26 | * @since 2.0 27 | */ 28 | public class NamedQuery extends NamedSchema { 29 | private static final String CONF_QUERY = "query"; 30 | private static final String CONF_SCHEMA = "schema"; 31 | private static final String CONF_PARAMETERS = "parameters"; 32 | 33 | private final String query; 34 | private final String schema; 35 | 36 | private final QueryParameters parameters; 37 | 38 | @SuppressWarnings("unchecked") 39 | public static NamedQuery newInstance(Object... args) { 40 | if (Objects.requireNonNull(args).length < 2) { 41 | throw new IllegalArgumentException( 42 | "In order to create named query, you need to specify at least ID and repository."); 43 | } 44 | 45 | String id = (String) args[0]; 46 | Repository manager = (Repository) Objects.requireNonNull(args[1]); 47 | JsonObject config = args.length > 2 ? (JsonObject) args[2] : null; 48 | 49 | NamedQuery query = new NamedQuery(id, manager, config); 50 | query.validate(); 51 | 52 | return query; 53 | } 54 | 55 | public NamedQuery(String id, Repository repo, JsonObject config) { 56 | super(id, repo, config); 57 | 58 | String str = config.getString(CONF_QUERY); 59 | this.query = Objects.requireNonNull(str); 60 | str = config.getString(CONF_SCHEMA); 61 | this.schema = str == null ? Utils.EMPTY_STRING : str; 62 | 63 | this.parameters = new QueryParameters(config.getJsonObject(CONF_PARAMETERS)); 64 | } 65 | 66 | public String getQuery() { 67 | return this.query; 68 | } 69 | 70 | public String getSchema() { 71 | return this.schema; 72 | } 73 | 74 | public QueryParameters getParameters() { 75 | return this.parameters; 76 | } 77 | } -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/NamedSchema.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.util.Objects; 19 | 20 | import io.vertx.core.json.JsonObject; 21 | 22 | /** 23 | * This class defines a named schema. 24 | * 25 | * @since 2.0 26 | */ 27 | public class NamedSchema extends ManagedEntity { 28 | protected static final String CONF_COLUMNS = "columns"; 29 | 30 | private final TableDefinition columns; 31 | 32 | @SuppressWarnings("unchecked") 33 | public static NamedSchema newInstance(Object... args) { 34 | if (Objects.requireNonNull(args).length < 2) { 35 | throw new IllegalArgumentException( 36 | "In order to create named schema, you need to specify at least ID and repository."); 37 | } 38 | 39 | String id = (String) args[0]; 40 | Repository manager = (Repository) Objects.requireNonNull(args[1]); 41 | JsonObject config = args.length > 2 ? (JsonObject) args[2] : null; 42 | 43 | NamedSchema schema = new NamedSchema(id, manager, config); 44 | schema.validate(); 45 | 46 | return schema; 47 | } 48 | 49 | public NamedSchema(String id, Repository repo, JsonObject config) { 50 | super(id, Objects.requireNonNull(config)); 51 | 52 | this.columns = TableDefinition.fromJson(config.getJsonArray(CONF_COLUMNS)); 53 | } 54 | 55 | public boolean hasColumn() { 56 | return this.columns != null && this.columns.hasColumn(); 57 | } 58 | 59 | public TableDefinition getColumns() { 60 | return this.columns; 61 | } 62 | 63 | @Override 64 | public UsageStats getUsage(String idOrAlias) { 65 | return null; 66 | } 67 | } -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/Reloadable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import io.vertx.core.json.JsonObject; 19 | 20 | /** 21 | * This interface defines the ability to load configuration automatically when 22 | * there's change detected. 23 | * 24 | * @since 2.0 25 | */ 26 | public interface Reloadable { 27 | /** 28 | * Reload configuration. This will be called for first-time loading and reload 29 | * when there's changes. 30 | * 31 | * @param config configuration in JSON format 32 | */ 33 | default void reload(JsonObject config) { 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/Repository.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * This interface defines a repository for managing {@link ManagedEntity} like 22 | * {@link NamedDataSource}, {@link NamedSchema}, and {@link NamedQuery}. 23 | * 24 | * @since 2.0 25 | */ 26 | public interface Repository { 27 | /** 28 | * Get class of managed entities. 29 | * 30 | * @return class of managed entities 31 | */ 32 | Class getEntityClass(); 33 | 34 | /** 35 | * Check if the given type of entity can be managed by this repository. 36 | * 37 | * @param clazz class of the entity 38 | * @return true if the type of entity can be managed by this repository; false 39 | * otherwise 40 | */ 41 | boolean accept(Class clazz); 42 | 43 | /** 44 | * Resolve given name. Usually just about DNS SRV record resolving, for example: 45 | * {@code jdbc:clickhouse:{{ ch-server.somedomain }}/system} will be resolved to 46 | * something like {@code jdbc:clickhouse:127.0.0.1:8123/system}. 47 | * 48 | * @param name name to resolve 49 | * @return resolved name 50 | */ 51 | String resolve(String name); 52 | 53 | /** 54 | * Get usage statistics of the repository. 55 | * 56 | * @return usage statistics 57 | */ 58 | List getUsageStats(); 59 | 60 | /** 61 | * Register new type of entity to be manged in this repository. 62 | * 63 | * @param type type of entity, defaults to extension class name 64 | * @param extension extension for instantiate new type of entity 65 | */ 66 | void registerType(String type, Extension extension); 67 | 68 | /** 69 | * Put a named entity into the repository. 70 | * 71 | * @param id id of the entity, could be null 72 | * @param entity non-null entity to be added 73 | */ 74 | void put(String id, T entity); 75 | 76 | /** 77 | * Get entity from repository by id. 78 | * 79 | * @param id id of the entity 80 | * @return desired entity 81 | */ 82 | T get(String id); 83 | 84 | /** 85 | * Get or create entity from repository by id. 86 | * 87 | * @param id id of the entity 88 | * @return desired entity 89 | */ 90 | default T getOrCreate(String id) { 91 | T entity = get(id); 92 | 93 | if (entity == null) { 94 | throw new UnsupportedOperationException("Creating entity is not supported"); 95 | } 96 | 97 | return entity; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/RepositoryManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * This interface defines a service for managing entities like 22 | * {@link NamedDataSource}, {@link NamedSchema}, and {@link NamedQuery}. 23 | * 24 | * @since 2.0 25 | */ 26 | public interface RepositoryManager { 27 | /** 28 | * Get repository capable of managing given type of entity. 29 | * 30 | * @param type of entity to be managed 31 | * @param clazz class of entity 32 | * @return repository capable of managing given type of entity 33 | */ 34 | Repository getRepository(Class clazz); 35 | 36 | /** 37 | * Update repository list using given repositories. 38 | * 39 | * @param repos repositories 40 | */ 41 | void update(List> repos); 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/ResponseWriter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import io.vertx.core.Handler; 19 | import io.vertx.core.http.HttpServerResponse; 20 | 21 | /** 22 | * This class defines how we write data to http response. 23 | * 24 | * @since 2.0 25 | */ 26 | public class ResponseWriter { 27 | private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(ResponseWriter.class); 28 | 29 | private final HttpServerResponse response; 30 | private final StreamOptions options; 31 | private final long timeout; 32 | 33 | private final long startTime; 34 | 35 | public ResponseWriter(HttpServerResponse response, StreamOptions options, int timeout) { 36 | this.response = response; 37 | this.options = options; 38 | this.timeout = timeout * 1000L; 39 | 40 | this.startTime = System.currentTimeMillis(); 41 | 42 | this.response.setWriteQueueMaxSize(this.options.getMaxBlockSize()); 43 | 44 | if (log.isDebugEnabled()) { 45 | log.debug("Start Time={}, Timeout={}, Max Block Size={}", this.startTime, this.timeout, 46 | this.options.getMaxBlockSize()); 47 | } 48 | } 49 | 50 | public StreamOptions getOptions() { 51 | return this.options; 52 | } 53 | 54 | public boolean isOpen() { 55 | return !this.response.closed() && !this.response.ended(); 56 | } 57 | 58 | public void setDrainHanlder(Handler handler) { 59 | this.response.drainHandler(handler); 60 | } 61 | 62 | public void write(ByteBuffer buffer) { 63 | if (this.response.closed() || this.response.ended()) { 64 | if (buffer != null && buffer.length() > 0) { 65 | log.warn("Still have at least {} bytes in buffer", buffer.length()); 66 | } 67 | 68 | throw new IllegalStateException("Response stream was closed"); 69 | } 70 | 71 | if (this.timeout > 0 && ((System.currentTimeMillis() - this.startTime) > this.timeout)) { 72 | throw new IllegalStateException("Abort due to timeout"); 73 | } 74 | 75 | this.response.write(buffer.unwrap()); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/StreamOptions.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import io.vertx.core.MultiMap; 19 | 20 | /** 21 | * This class defines options for streaming. 22 | * 23 | * @since 2.0 24 | */ 25 | public class StreamOptions { 26 | private static final String PARAM_MAX_BLOCK_SIZE = "max_block_size"; 27 | 28 | public static final int DEFAULT_BLOCK_SIZE = 65535; 29 | 30 | private final int maxBlockSize; 31 | 32 | public StreamOptions(MultiMap params) { 33 | int blockSize = DEFAULT_BLOCK_SIZE; 34 | try { 35 | blockSize = Integer.parseInt(params.get(PARAM_MAX_BLOCK_SIZE)); 36 | } catch (Exception e) { 37 | } 38 | 39 | this.maxBlockSize = blockSize; 40 | } 41 | 42 | public int getMaxBlockSize() { 43 | return this.maxBlockSize; 44 | } 45 | } -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/core/UsageStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | /** 19 | * This interface defines the ability to load configuration automatically when 20 | * there's change detected. 21 | * 22 | * @since 2.0 23 | */ 24 | public interface UsageStats { 25 | String getName(); 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/impl/DefaultDataTypeConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.impl; 17 | 18 | import java.math.BigDecimal; 19 | import java.math.BigInteger; 20 | import java.sql.JDBCType; 21 | import java.util.List; 22 | 23 | import com.clickhouse.jdbcbridge.core.DataType; 24 | import com.clickhouse.jdbcbridge.core.DataTypeConverter; 25 | import com.clickhouse.jdbcbridge.core.DataTypeMapping; 26 | 27 | /** 28 | * This class is default implementation of DataTypeConvert. 29 | * 30 | * @since 2.0 31 | */ 32 | public class DefaultDataTypeConverter implements DataTypeConverter { 33 | private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(DataTypeConverter.class); 34 | 35 | private final DataTypeMapping[] mappings; 36 | 37 | public DefaultDataTypeConverter() { 38 | this(null); 39 | } 40 | 41 | public DefaultDataTypeConverter(List mappings) { 42 | if (mappings == null) { 43 | this.mappings = new DataTypeMapping[0]; 44 | } else { 45 | this.mappings = new DataTypeMapping[mappings.size()]; 46 | int index = 0; 47 | for (DataTypeMapping m : mappings) { 48 | this.mappings[index++] = m; 49 | } 50 | } 51 | } 52 | 53 | @Override 54 | public DataType from(JDBCType jdbcType, String typeName, int precision, int scale, boolean signed) { 55 | for (int i = 0; i < mappings.length; i++) { 56 | if (mappings[i].accept(jdbcType, typeName)) { 57 | return mappings[i].getMappedType(); 58 | } 59 | } 60 | 61 | DataType type = DataType.Str; 62 | 63 | switch (jdbcType) { 64 | case BIT: 65 | if (precision > 128) { 66 | type = DataType.Int256; 67 | } else if (precision > 64) { 68 | type = DataType.Int128; 69 | } else if (precision > 32) { 70 | type = DataType.Int64; 71 | } else if (precision > 16) { 72 | type = DataType.Int32; 73 | } else if (precision > 8) { 74 | type = DataType.Int16; 75 | } else { 76 | type = DataType.Int8; 77 | } 78 | break; 79 | case TINYINT: 80 | type = signed ? DataType.Int8 : DataType.UInt8; 81 | break; 82 | case SMALLINT: 83 | type = signed ? DataType.Int16 : DataType.UInt16; 84 | break; 85 | case INTEGER: 86 | type = signed ? DataType.Int32 : DataType.UInt32; 87 | break; 88 | case BIGINT: 89 | type = signed ? DataType.Int64 : DataType.UInt64; 90 | break; 91 | case REAL: 92 | case FLOAT: 93 | type = DataType.Float32; 94 | break; 95 | case DOUBLE: 96 | type = DataType.Float64; 97 | break; 98 | case NUMERIC: 99 | case DECIMAL: 100 | type = DataType.Decimal; 101 | break; 102 | case ARRAY: 103 | case OTHER: 104 | case BOOLEAN: 105 | case CHAR: 106 | case NCHAR: 107 | case VARCHAR: 108 | case NVARCHAR: 109 | case LONGVARCHAR: 110 | case LONGNVARCHAR: 111 | case NULL: 112 | type = DataType.Str; 113 | break; 114 | case DATE: 115 | type = DataType.Date; // precision should be 10 116 | break; 117 | case TIME: 118 | case TIMESTAMP: 119 | case TIME_WITH_TIMEZONE: 120 | case TIMESTAMP_WITH_TIMEZONE: 121 | // type = useDateTime ? DataType.DateTime : DataType.DateTime64; 122 | type = scale > 0 ? DataType.DateTime64 : DataType.DateTime; 123 | break; 124 | default: 125 | log.warn("Unsupported JDBC type [{}], which will be treated as [{}]", jdbcType.name(), type.name()); 126 | break; 127 | } 128 | 129 | return type; 130 | } 131 | 132 | @Override 133 | public DataType from(Object javaObject) { 134 | final DataType type; 135 | 136 | if (javaObject == null) { 137 | type = DataType.Str; 138 | } else if (javaObject instanceof Byte) { 139 | type = DataType.Int8; 140 | } else if (javaObject instanceof Short) { 141 | type = DataType.Int16; 142 | } else if (javaObject instanceof Integer) { 143 | type = DataType.Int32; 144 | } else if (javaObject instanceof Long) { 145 | type = DataType.Int64; 146 | } else if (javaObject instanceof BigInteger) { 147 | type = DataType.Int256; 148 | } else if (javaObject instanceof Float) { 149 | type = DataType.Float32; 150 | } else if (javaObject instanceof Double) { 151 | type = DataType.Float64; 152 | } else if (javaObject instanceof BigDecimal) { 153 | type = DataType.Decimal256; 154 | } else { 155 | type = DataType.Str; 156 | } 157 | 158 | return type; 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/impl/DefaultRepositoryManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.impl; 17 | 18 | import java.util.ArrayList; 19 | import java.util.Collections; 20 | import java.util.List; 21 | import java.util.Objects; 22 | 23 | import com.clickhouse.jdbcbridge.core.ManagedEntity; 24 | import com.clickhouse.jdbcbridge.core.Repository; 25 | import com.clickhouse.jdbcbridge.core.RepositoryManager; 26 | 27 | /** 28 | * Default implementation of 29 | * {@link com.clickhouse.jdbcbridge.core.RepositoryManager}. 30 | * 31 | * @since 2.0 32 | */ 33 | public class DefaultRepositoryManager implements RepositoryManager { 34 | private final List> repos = Collections.synchronizedList(new ArrayList<>()); 35 | 36 | @SuppressWarnings("unchecked") 37 | @Override 38 | public Repository getRepository(Class clazz) { 39 | Objects.requireNonNull(clazz); 40 | 41 | for (Repository repo : repos) { 42 | if (repo.accept(clazz)) { 43 | return (Repository) repo; 44 | } 45 | } 46 | 47 | throw new IllegalArgumentException("No repository available for " + clazz.getName()); 48 | } 49 | 50 | @Override 51 | public void update(List> repos) { 52 | if (repos == null) { 53 | return; 54 | } 55 | 56 | for (Repository repo : repos) { 57 | boolean replaced = false; 58 | for (int i = 0, len = this.repos.size(); i < len; i++) { 59 | Repository current = this.repos.get(i); 60 | if (!current.getClass().equals(repo.getClass()) 61 | && !current.getEntityClass().equals(repo.getEntityClass())) { 62 | this.repos.set(i, repo); 63 | replaced = true; 64 | break; 65 | } 66 | } 67 | 68 | if (!replaced) { 69 | this.repos.add(repo); 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/clickhouse/jdbcbridge/impl/JsonFileRepository.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.impl; 17 | 18 | import java.util.HashSet; 19 | import java.util.Objects; 20 | import java.util.Map.Entry; 21 | 22 | import com.clickhouse.jdbcbridge.core.BaseRepository; 23 | import com.clickhouse.jdbcbridge.core.ExtensionManager; 24 | import com.clickhouse.jdbcbridge.core.ManagedEntity; 25 | import com.clickhouse.jdbcbridge.core.NamedDataSource; 26 | import com.clickhouse.jdbcbridge.core.NamedQuery; 27 | import com.clickhouse.jdbcbridge.core.NamedSchema; 28 | import com.clickhouse.jdbcbridge.core.Reloadable; 29 | import com.clickhouse.jdbcbridge.core.Utils; 30 | 31 | import io.vertx.core.json.JsonObject; 32 | 33 | public class JsonFileRepository extends BaseRepository implements Reloadable { 34 | private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(JsonFileRepository.class); 35 | 36 | @SuppressWarnings("unchecked") 37 | public static JsonFileRepository newInstance(Object... args) { 38 | if (Objects.requireNonNull(args).length < 2) { 39 | throw new IllegalArgumentException( 40 | "In order to create a JSON file repository, you need to specify at least ExtensionManager and entity class."); 41 | } 42 | 43 | ExtensionManager manager = (ExtensionManager) Objects.requireNonNull(args[0]); 44 | Class entityClass = (Class) Objects.requireNonNull(args[1]); 45 | 46 | JsonFileRepository repo = new JsonFileRepository<>(entityClass); 47 | String defaultDir = entityClass.getSimpleName().toLowerCase(); 48 | String defaultEnv = entityClass.getSimpleName().toUpperCase() + "_CONFIG_DIR"; 49 | String defaultProp = "jdbc-bridge." + defaultDir + ".config.dir"; 50 | if (NamedDataSource.class.equals(entityClass)) { 51 | defaultDir = "datasources"; 52 | defaultEnv = "DATASOURCE_CONFIG_DIR"; 53 | defaultProp = "jdbc-bridge.datasource.config.dir"; 54 | } else if (NamedSchema.class.equals(entityClass)) { 55 | defaultDir = "schemas"; 56 | defaultEnv = "SCHEMA_CONFIG_DIR"; 57 | defaultProp = "jdbc-bridge.schema.config.dir"; 58 | } else if (NamedQuery.class.equals(entityClass)) { 59 | defaultDir = "queries"; 60 | defaultEnv = "QUERY_CONFIG_DIR"; 61 | defaultProp = "jdbc-bridge.query.config.dir"; 62 | } 63 | 64 | manager.registerConfigLoader(Utils.getConfiguration(defaultDir, defaultEnv, defaultProp), repo::reload); 65 | 66 | return repo; 67 | } 68 | 69 | public JsonFileRepository(Class clazz) { 70 | super(clazz); 71 | } 72 | 73 | @Override 74 | public void reload(JsonObject config) { 75 | if (config == null || config.fieldNames().size() == 0) { 76 | log.info("No {} configuration found", getEntityName()); 77 | 78 | HashSet keys = new HashSet<>(); 79 | for (String key : mappings.keySet()) { 80 | keys.add(key); 81 | } 82 | 83 | for (String key : keys) { 84 | remove(key); 85 | } 86 | // mappings.clear(); 87 | } else { 88 | log.info("Loading {} configuration...", getEntityName()); 89 | HashSet keys = new HashSet<>(); 90 | for (Entry entry : config) { 91 | String key = entry.getKey(); 92 | Object value = entry.getValue(); 93 | if (key != null && value instanceof JsonObject) { 94 | keys.add(key); 95 | update(key, (JsonObject) value); 96 | } 97 | } 98 | 99 | HashSet entityIds = new HashSet<>(); 100 | mappings.entrySet().forEach(entry -> { 101 | String id = entry.getKey(); 102 | T entity = entry.getValue(); 103 | if (id != null && !id.isEmpty() && entity != null && id.equals(entity.getId())) { 104 | entityIds.add(id); 105 | } 106 | }); 107 | 108 | for (String id : entityIds) { 109 | if (!keys.contains(id)) { 110 | remove(id); 111 | } 112 | } 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/com.clickhouse.jdbcbridge.core.DataTypeConverter: -------------------------------------------------------------------------------- 1 | com.clickhouse.jdbcbridge.impl.DefaultDataTypeConverter -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/com.clickhouse.jdbcbridge.core.RepositoryManager: -------------------------------------------------------------------------------- 1 | com.clickhouse.jdbcbridge.impl.DefaultRepositoryManager -------------------------------------------------------------------------------- /src/main/resources/logging.properties: -------------------------------------------------------------------------------- 1 | handlers=java.util.logging.ConsoleHandler 2 | 3 | java.util.logging.ConsoleHandler.level=DEBUG 4 | java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter 5 | java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$-7s] %5$s %n 6 | 7 | .level=DEBUG 8 | -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/JdbcBridgeVerticleTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge; 17 | 18 | import static org.testng.Assert.*; 19 | 20 | import java.time.Duration; 21 | import java.util.List; 22 | 23 | import com.clickhouse.jdbcbridge.core.BaseRepository; 24 | import com.clickhouse.jdbcbridge.core.Extension; 25 | import com.clickhouse.jdbcbridge.core.ExtensionManager; 26 | import com.clickhouse.jdbcbridge.core.ManagedEntity; 27 | import com.clickhouse.jdbcbridge.core.Utils; 28 | 29 | import org.testcontainers.containers.BindMode; 30 | import org.testcontainers.containers.GenericContainer; 31 | import org.testcontainers.containers.Network; 32 | import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; 33 | import org.testcontainers.containers.wait.strategy.Wait; 34 | import org.testng.annotations.AfterSuite; 35 | import org.testng.annotations.BeforeSuite; 36 | import org.testng.annotations.Test; 37 | 38 | import static java.time.temporal.ChronoUnit.SECONDS; 39 | 40 | public class JdbcBridgeVerticleTest { 41 | // containers for SIT(sytem integration test) 42 | private static final Network sharedNetwork = Network.newNetwork(); 43 | // https://github.com/testcontainers/testcontainers-java/blob/master/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java 44 | private static final GenericContainer pgServer = new GenericContainer<>("postgres:11.10-alpine") 45 | .withNetwork(sharedNetwork).withNetworkAliases("postgresql_server").withEnv("POSTGRES_DB", "test") 46 | .withEnv("POSTGRES_USER", "sa").withEnv("POSTGRES_PASSWORD", "sa") 47 | .waitingFor(new LogMessageWaitStrategy().withRegEx(".*database system is ready to accept connections.*\\s") 48 | .withTimes(2).withStartupTimeout(Duration.of(60, SECONDS))); 49 | // https://github.com/testcontainers/testcontainers-java/blob/master/modules/mariadb/src/main/java/org/testcontainers/containers/MariaDBContainer.java 50 | private static final GenericContainer mdServer = new GenericContainer<>("mariadb:10.5") 51 | .withNetwork(sharedNetwork).withNetworkAliases("mariadb_server").withEnv("MYSQL_DATABASE", "test") 52 | .withEnv("MYSQL_ROOT_PASSWORD", "root").withStartupAttempts(3) 53 | .waitingFor(new LogMessageWaitStrategy().withRegEx(".*mysqld: ready for connections.*").withTimes(2) 54 | .withStartupTimeout(Duration.of(60, SECONDS))); 55 | 56 | private static final GenericContainer jbServer = new GenericContainer<>("clickhouse/jdbc-bridge") 57 | .withNetwork(sharedNetwork).withNetworkAliases("jdbc_bridge_server") 58 | .withFileSystemBind("target", "/build", BindMode.READ_WRITE) 59 | .withWorkingDirectory("/build/test-classes/sit/jdbc-bridge") 60 | .withCommand("bash", "-c", "java -jar /build/clickhouse-jdbc-bridge*.jar") 61 | .waitingFor(Wait.forHttp("/ping").forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); 62 | private static final GenericContainer chServer = new GenericContainer<>("clickhouse/clickhouse-server:22.3") 63 | .withNetwork(sharedNetwork).withNetworkAliases("clickhouse_server") 64 | .withClasspathResourceMapping("sit/ch-server/jdbc-bridge.xml", 65 | "/etc/clickhouse-server/config.d/jdbc-bridge.xml", BindMode.READ_ONLY) 66 | .waitingFor(Wait.forHttp("/ping").forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); 67 | 68 | public static class TestRepository extends BaseRepository { 69 | public TestRepository(ExtensionManager manager, Class clazz) { 70 | super(clazz); 71 | } 72 | } 73 | 74 | @BeforeSuite(groups = { "sit" }) 75 | public static void beforeSuite() { 76 | pgServer.start(); 77 | mdServer.start(); 78 | chServer.start(); 79 | jbServer.start(); 80 | } 81 | 82 | @AfterSuite(groups = { "sit" }) 83 | public static void afterSuite() { 84 | pgServer.stop(); 85 | mdServer.stop(); 86 | chServer.stop(); 87 | jbServer.stop(); 88 | } 89 | 90 | @Test(groups = { "unit" }) 91 | public void testLoadExtensions() { 92 | JdbcBridgeVerticle main = new JdbcBridgeVerticle(); 93 | List> extensions = main.loadExtensions(null); 94 | assertNotNull(extensions); 95 | assertEquals(extensions.size(), 3); 96 | 97 | extensions = main.loadExtensions(Utils.loadJsonFromFile("src/test/resources/server.json")); 98 | assertNotNull(extensions); 99 | assertEquals(extensions.size(), 3); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/core/DataTypeMappingTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import static org.testng.Assert.*; 19 | 20 | import java.sql.JDBCType; 21 | import java.sql.Types; 22 | 23 | import org.testng.annotations.Test; 24 | 25 | public class DataTypeMappingTest { 26 | @Test(groups = { "unit" }) 27 | public void testConstructor() { 28 | DataTypeMapping m = new DataTypeMapping(Types.BOOLEAN, null, DataType.Bool); 29 | assertEquals(m.getSourceJdbcType(), JDBCType.BOOLEAN); 30 | assertEquals(m.getSourceNativeType(), null); 31 | assertTrue(m.accept(JDBCType.BOOLEAN, null)); 32 | assertEquals(m.getMappedType(), DataType.Bool); 33 | 34 | m = new DataTypeMapping("boolean", "bool", "String"); 35 | assertEquals(m.getSourceJdbcType(), JDBCType.BOOLEAN); 36 | assertEquals(m.getSourceNativeType(), "bool"); 37 | assertFalse(m.accept(JDBCType.BOOLEAN, null)); 38 | assertFalse(m.accept(JDBCType.BOOLEAN, "Bool")); 39 | assertTrue(m.accept(JDBCType.VARCHAR, "bool")); 40 | assertEquals(m.getMappedType(), DataType.Str); 41 | 42 | m = new DataTypeMapping("bit", "*", "Int8"); 43 | assertEquals(m.getSourceJdbcType(), JDBCType.BIT); 44 | assertEquals(m.getSourceNativeType(), "*"); 45 | assertTrue(m.getSourceNativeType() == DataTypeMapping.ANY_NATIVE_TYPE); 46 | assertTrue(m.accept(JDBCType.BOOLEAN, null)); 47 | assertTrue(m.accept(JDBCType.BIT, "Bool")); 48 | assertTrue(m.accept(JDBCType.VARCHAR, "bit")); 49 | assertEquals(m.getMappedType(), DataType.Int8); 50 | } 51 | } -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/core/DnsResolverTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import static org.testng.Assert.*; 19 | 20 | import org.testng.annotations.Test; 21 | 22 | public class DnsResolverTest { 23 | @Test(groups = { "unit" }) 24 | public void testResolve() { 25 | String dns = "_sip._udp.sip.voice.google.com"; 26 | assertNotNull(new DnsResolver().resolve(dns, true)); 27 | assertNotNull(new DnsResolver().resolve(dns, false)); 28 | 29 | dns = "_sip._udp.sip.voice.google.com."; 30 | assertNotNull(new DnsResolver().resolve(dns, true)); 31 | assertNotNull(new DnsResolver().resolve(dns, false)); 32 | } 33 | 34 | @Test(groups = { "unit" }) 35 | public void testApply() { 36 | String host = "_sip._udp.sip.voice.google.com"; 37 | String port = "5060"; 38 | String hostAndPort = host + ":" + port; 39 | 40 | String dns = "_sip._udp.sip.voice.google.com"; 41 | assertEquals(new DnsResolver().apply(dns), hostAndPort); 42 | assertEquals(new DnsResolver().apply("host:" + dns), host); 43 | assertEquals(new DnsResolver().apply("port:" + dns), port); 44 | 45 | dns = "_sip._udp.sip.voice.google.com."; 46 | assertEquals(new DnsResolver().apply(dns), hostAndPort); 47 | assertEquals(new DnsResolver().apply("host:" + dns), host); 48 | assertEquals(new DnsResolver().apply("port:" + dns), port); 49 | } 50 | } -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/core/ExtensionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import static org.testng.Assert.*; 19 | 20 | import java.util.Objects; 21 | 22 | import org.testng.annotations.Test; 23 | 24 | public class ExtensionTest { 25 | static abstract class MyService { 26 | public abstract String echo(String input); 27 | } 28 | 29 | static class EchoService extends MyService { 30 | public EchoService(Object someContext) { 31 | } 32 | 33 | @Override 34 | public String echo(String input) { 35 | return input; 36 | } 37 | } 38 | 39 | static class MyExtension extends MyService { 40 | public static final String EXTENSION_NAME = "mine"; 41 | 42 | public static final String DEFAULT_PREFIX = "unknown"; 43 | public static boolean flag = false; 44 | 45 | public static void initialize(ExtensionManager manager) { 46 | flag = true; 47 | } 48 | 49 | public static MyExtension newInstance(Object... args) { 50 | return new MyExtension(args != null && args.length > 0 ? Objects.toString(args[0]) : DEFAULT_PREFIX); 51 | } 52 | 53 | private final String prefix; 54 | 55 | public MyExtension(String prefix) { 56 | this.prefix = Objects.requireNonNull(prefix); 57 | } 58 | 59 | @Override 60 | public String echo(String input) { 61 | return this.prefix + input; 62 | } 63 | } 64 | 65 | static class AnotherExtension extends MyService { 66 | public static final String DEFAULT_PREFIX = "another"; 67 | 68 | private final String prefix; 69 | 70 | public AnotherExtension(byte prefix) { 71 | this.prefix = String.valueOf(prefix); 72 | } 73 | 74 | public AnotherExtension(int prefix) { 75 | this.prefix = String.valueOf(prefix); 76 | } 77 | 78 | public AnotherExtension(long prefix) { 79 | this.prefix = String.valueOf(prefix); 80 | } 81 | 82 | public AnotherExtension(String prefix) { 83 | this.prefix = Objects.requireNonNull(prefix); 84 | } 85 | 86 | @Override 87 | public String echo(String input) { 88 | return this.prefix + input; 89 | } 90 | } 91 | 92 | @Test(groups = { "unit" }) 93 | public void testExtension() { 94 | assertFalse(MyExtension.flag); 95 | 96 | Extension extension = new Extension<>(MyExtension.class); 97 | assertEquals(extension.getName(), MyExtension.EXTENSION_NAME); 98 | assertEquals(extension.getProviderClass(), MyExtension.class); 99 | 100 | extension.initialize(null); 101 | 102 | assertTrue(MyExtension.flag); 103 | 104 | String input = "input"; 105 | MyService service = extension.newInstance(); 106 | assertTrue(service instanceof MyExtension); 107 | assertEquals(((MyExtension) service).prefix, MyExtension.DEFAULT_PREFIX); 108 | assertEquals(((MyExtension) service).echo(input), MyExtension.DEFAULT_PREFIX + input); 109 | 110 | String prefix = "prefix"; 111 | service = extension.newInstance(prefix); 112 | assertTrue(service instanceof MyExtension); 113 | assertEquals(((MyExtension) service).prefix, prefix); 114 | assertEquals(((MyExtension) service).echo(input), prefix + input); 115 | } 116 | 117 | @Test(groups = { "unit" }) 118 | public void testNotFullyImplementedExtension() { 119 | Extension extension = new Extension<>(EchoService.class); 120 | assertEquals(extension.getName(), EchoService.class.getSimpleName()); 121 | assertEquals(extension.getProviderClass(), EchoService.class); 122 | 123 | extension.initialize(null); 124 | 125 | assertThrows(UnsupportedOperationException.class, extension::newInstance); 126 | 127 | String prefix = "secret"; 128 | final Extension ext = new Extension<>(AnotherExtension.class); 129 | ext.initialize(null); 130 | assertThrows(UnsupportedOperationException.class, ext::newInstance); 131 | assertThrows(UnsupportedOperationException.class, new ThrowingRunnable() { 132 | @Override 133 | public void run() throws Throwable { 134 | ext.newInstance(null); 135 | } 136 | }); 137 | assertThrows(UnsupportedOperationException.class, new ThrowingRunnable() { 138 | @Override 139 | public void run() throws Throwable { 140 | ext.newInstance(1.1); 141 | } 142 | }); 143 | assertThrows(UnsupportedOperationException.class, new ThrowingRunnable() { 144 | @Override 145 | public void run() throws Throwable { 146 | ext.newInstance(ext); 147 | } 148 | }); 149 | assertThrows(IllegalArgumentException.class, new ThrowingRunnable() { 150 | @Override 151 | public void run() throws Throwable { 152 | ext.newInstance(new Object[] { null }); 153 | } 154 | }); 155 | 156 | MyService service = ext.newInstance(new Object[] { 1 }); 157 | assertTrue(service instanceof AnotherExtension); 158 | assertEquals(((AnotherExtension) service).prefix, "1"); 159 | 160 | service = ext.newInstance(prefix); 161 | assertTrue(service instanceof AnotherExtension); 162 | assertEquals(((AnotherExtension) service).prefix, prefix); 163 | } 164 | } -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/core/NamedDataSourceTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import static org.testng.Assert.*; 19 | 20 | import org.testng.annotations.Test; 21 | 22 | import io.vertx.core.json.JsonObject; 23 | 24 | public class NamedDataSourceTest { 25 | static class TestRepository extends BaseRepository { 26 | private int counter = 0; 27 | 28 | public TestRepository(Class clazz) { 29 | super(clazz); 30 | } 31 | 32 | @Override 33 | protected void atomicAdd(T entity) { 34 | counter++; 35 | } 36 | 37 | @Override 38 | protected void atomicRemove(T entity) { 39 | counter--; 40 | } 41 | 42 | public int getCounter() { 43 | return this.counter; 44 | } 45 | } 46 | 47 | @Test(groups = { "unit" }) 48 | public void testConstructor() { 49 | String dataSourceId = "test-datasource"; 50 | JsonObject config = Utils.loadJsonFromFile("src/test/resources/datasources/test-datasource.json"); 51 | 52 | NamedDataSource ds = new NamedDataSource(dataSourceId, new TestRepository<>(NamedDataSource.class), 53 | config.getJsonObject(dataSourceId)); 54 | assertEquals(ds.getId(), dataSourceId); 55 | for (ColumnDefinition col : ds.getCustomColumns()) { 56 | assertEquals(col.getName(), "c_" + col.getType().name().toLowerCase()); 57 | switch (col.getType()) { 58 | case Bool: 59 | assertEquals(String.valueOf(col.getValue()), "1"); 60 | break; 61 | case Decimal: 62 | case Decimal32: 63 | case Decimal64: 64 | case Decimal128: 65 | case Decimal256: 66 | case Float32: 67 | case Float64: 68 | assertEquals(String.valueOf(col.getValue()), "2.0", col.getType().name()); 69 | break; 70 | default: 71 | assertEquals(String.valueOf(col.getValue()), "2", col.getType().name()); 72 | break; 73 | } 74 | } 75 | 76 | for (DataType type : DataType.values()) { 77 | Object value = ds.getDefaultValues().getTypedValue(type).getValue(); 78 | switch (type) { 79 | case Bool: 80 | assertEquals(String.valueOf(value), "0"); 81 | break; 82 | case Decimal: 83 | case Decimal32: 84 | case Decimal64: 85 | case Decimal128: 86 | case Decimal256: 87 | case Float32: 88 | case Float64: 89 | assertEquals(String.valueOf(value), "3.0", type.name()); 90 | break; 91 | default: 92 | assertEquals(String.valueOf(value), "3", type.name()); 93 | break; 94 | } 95 | } 96 | } 97 | 98 | @Test(groups = { "unit" }) 99 | public void testGetColumns() { 100 | String dataSourceId = "test-datasource"; 101 | JsonObject config = Utils.loadJsonFromFile("src/test/resources/datasources/test-datasource.json"); 102 | 103 | NamedDataSource ds = new NamedDataSource(dataSourceId, new TestRepository<>(NamedDataSource.class), 104 | config.getJsonObject(dataSourceId)); 105 | ds.getResultColumns("", "src/test/resources/simple.query", new QueryParameters()); 106 | assertEquals(ds.getId(), dataSourceId); 107 | } 108 | } -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/core/QueryParametersTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import static org.testng.Assert.*; 19 | 20 | import org.testng.annotations.Test; 21 | 22 | public class QueryParametersTest { 23 | @Test(groups = { "unit" }) 24 | public void testMergeUri() { 25 | QueryParameters params = new QueryParameters(); 26 | 27 | assertEquals(params.isDebug(), false); 28 | assertEquals(params.nullAsDefault(), false); 29 | assertEquals(params.showCustomColumns(), false); 30 | assertEquals(params.showDatasourceColumn(), false); 31 | 32 | params.merge("ds?" + QueryParameters.PARAM_NULL_AS_DEFAULT); 33 | assertEquals(params.nullAsDefault(), true); 34 | params.merge("ds?" + QueryParameters.PARAM_CUSTOM_COLUMNS); 35 | assertEquals(params.showCustomColumns(), true); 36 | params.merge("ds?" + QueryParameters.PARAM_DATASOURCE_COLUMN + "&" + QueryParameters.PARAM_DEBUG); 37 | assertEquals(params.showDatasourceColumn(), true); 38 | assertEquals(params.isDebug(), true); 39 | } 40 | } -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/core/QueryParserTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import static org.testng.Assert.*; 19 | 20 | import org.testng.annotations.Test; 21 | 22 | public class QueryParserTest { 23 | @Test(groups = { "unit" }) 24 | public void testNormalizeQuery() { 25 | String query = "some_table"; 26 | assertEquals(QueryParser.normalizeQuery(query), query); 27 | 28 | query = "some named query"; 29 | assertEquals(QueryParser.normalizeQuery(query), query); 30 | 31 | query = "SELECT col1, col2 FROM some_table"; 32 | assertEquals(QueryParser.normalizeQuery(query), query); 33 | 34 | query = "SELECT col1, col2 FROM some_schema.some_table"; 35 | assertEquals(QueryParser.normalizeQuery(query), query); 36 | 37 | query = "SELECT `col1`, `col2` FROM `some_table`"; 38 | assertEquals(QueryParser.normalizeQuery(query), "some_table"); 39 | 40 | query = "SELECT `col1`, `col2` FROM `some_schema`.`some_table`"; 41 | assertEquals(QueryParser.normalizeQuery(query), "some_table"); 42 | 43 | query = "SELECT \"col1\", \"col2\" FROM \"some_table\""; 44 | assertEquals(QueryParser.normalizeQuery(query), "some_table"); 45 | 46 | query = "SELECT \"col1\", \"col2\" FROM \"some_schema\".\"some_table\""; 47 | assertEquals(QueryParser.normalizeQuery(query), "some_table"); 48 | 49 | String embeddedQuery = "select 1"; 50 | query = "SELECT `col1`, `col2` FROM `" + embeddedQuery + "`"; 51 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 52 | 53 | query = "SELECT `col1`, `col2` FROM `some_schema`.`" + embeddedQuery + "`"; 54 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 55 | 56 | query = "SELECT \"col1\", \"col2\" FROM \"" + embeddedQuery + "\""; 57 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 58 | 59 | query = "SELECT \"col1\", \"col2\" FROM \"some_schema\".\"" + embeddedQuery + "\""; 60 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 61 | 62 | embeddedQuery = "select 's' as s"; 63 | 64 | query = "SELECT `s` FROM `" + embeddedQuery + "`"; 65 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 66 | 67 | query = "SELECT `s` FROM `" + embeddedQuery + "` WHERE `s` = 's'"; 68 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 69 | 70 | embeddedQuery = "select\t'1`2\"3'\r\n, -- `\"\n/* \"s` */'`''`'"; 71 | query = "SELECT `col1`, `col2` FROM `" + embeddedQuery + "`"; 72 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 73 | 74 | query = "SELECT `col1`, `col2` FROM `some_schema`.`" + embeddedQuery + "`"; 75 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 76 | 77 | query = "SELECT \"col1\", \"col2\" FROM \"" + embeddedQuery + "\""; 78 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 79 | 80 | query = "SELECT \"col1\", \"col2\" FROM \"some_schema\".\"" + embeddedQuery + "\""; 81 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery); 82 | 83 | query = "SELECT * FROM test.test_table"; 84 | assertEquals(QueryParser.normalizeQuery(query), query); 85 | 86 | embeddedQuery = "SELECT 1 as \\`a\\`, ''\\`2'' as \\`b\\`"; 87 | query = "SELECT `col1`, `col2` FROM `" + embeddedQuery + "`"; 88 | assertEquals(QueryParser.normalizeQuery(query), embeddedQuery.replace("\\", "")); 89 | } 90 | 91 | @Test(groups = { "unit" }) 92 | public void testExtractSchemaName() { 93 | assertEquals(QueryParser.extractSchemaName(null), ""); 94 | assertEquals(QueryParser.extractSchemaName(""), ""); 95 | assertEquals(QueryParser.extractSchemaName("a"), ""); 96 | assertEquals(QueryParser.extractSchemaName("a.a"), ""); 97 | 98 | String schema = "schema"; 99 | assertEquals(QueryParser.extractSchemaName("SELECT * FROM " + "`" + schema + "`.`table`"), schema); 100 | assertEquals(QueryParser.extractSchemaName("SELECT * FROM " + "\"" + schema + "\".\"table\""), schema); 101 | } 102 | 103 | @Test(groups = { "unit" }) 104 | public void testExtractTableName() { 105 | assertEquals(QueryParser.extractTableName(null), null); 106 | assertEquals(QueryParser.extractTableName(""), ""); 107 | assertEquals(QueryParser.extractTableName("a"), "a"); 108 | assertEquals(QueryParser.extractTableName("a.a"), "a.a"); 109 | 110 | String table = "`schema`.`table`"; 111 | assertEquals(QueryParser.extractTableName("SELECT * FROM " + table), table); 112 | assertEquals(QueryParser.extractTableName("SELECT * from " + table), table); 113 | assertEquals(QueryParser.extractTableName("SELECT * FROM " + table + " where col1=11"), table); 114 | assertEquals(QueryParser.extractTableName("SELECT * FROM\r" + table + " where col1=11"), table); 115 | assertEquals(QueryParser.extractTableName("SELECT * FROM (select col1 from " + table + " where col1=11) a"), 116 | table); 117 | assertEquals(QueryParser.extractTableName( 118 | "SELECT col1, ' from b' as a FROM (select col1 from " + table + " where col1=11) a"), table); 119 | } 120 | 121 | @Test(groups = { "unit" }) 122 | public void testQueryWithParameters() { 123 | QueryParser parser = new QueryParser("connection_uri?p1=table&p2=column&p3=value1&p4=value2", "", 124 | "select a from {{ p1 }} where {{p2}} = 2 and '{{ p3}}' != '{{ p4}}'", 125 | "columns format version: 1\n1 columns:\n`a` Int32", "RowBinary", "false", null); 126 | 127 | assertEquals(parser.getNormalizedQuery(), "select a from table where column = 2 and 'value1' != 'value2'"); 128 | } 129 | 130 | // Reference: 131 | // https://github.com/ClickHouse/ClickHouse/blob/06446b4f08a142d6f1bc30664c47ded88ab51782/src/Common/tests/gtest_unescapeForFileName.cpp#L9 132 | @Test(groups = { "unit" }) 133 | public void testUnescape() { 134 | String str = ""; 135 | assertEquals(str, QueryParser.unescape(str)); 136 | assertEquals(str = "172.19.0.6", QueryParser.unescape(str)); 137 | assertEquals(str = "abcd.", QueryParser.unescape(str)); 138 | assertEquals(str = "abcd", QueryParser.unescape(str)); 139 | assertEquals(str = "..::", QueryParser.unescape(str)); 140 | 141 | assertEquals("`nbusfreq` Nullable(String)", QueryParser.unescape("%60nbusfreq%60%20Nullable%28String%29")); 142 | assertEquals("里程", QueryParser.unescape("%E9%87%8C%E7%A8%8B")); 143 | 144 | assertEquals("{", QueryParser.unescape("%7b")); 145 | assertEquals("{", QueryParser.unescape("%7B")); 146 | assertEquals("09AFaf", QueryParser.unescape("%30%39%41%46%61%66")); 147 | } 148 | } -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/core/TableDefinitionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.core; 17 | 18 | import static com.clickhouse.jdbcbridge.core.DataType.*; 19 | import static org.testng.Assert.*; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.testng.annotations.Test; 25 | 26 | public class TableDefinitionTest { 27 | @Test(groups = { "unit" }) 28 | public void testUpdateValues() { 29 | TableDefinition list = new TableDefinition( 30 | new ColumnDefinition("column1", DataType.Int32, true, DEFAULT_LENGTH, DEFAULT_PRECISION, DEFAULT_SCALE), 31 | new ColumnDefinition("column2", DataType.Str, true, DEFAULT_LENGTH, DEFAULT_PRECISION, DEFAULT_SCALE), 32 | new ColumnDefinition("column3", DataType.Int32, true, DEFAULT_LENGTH, DEFAULT_PRECISION, 33 | DEFAULT_SCALE)); 34 | 35 | Object[] expectedValues = new Object[] { 0, "", 0 }; 36 | for (int i = 0; i < list.size(); i++) { 37 | assertEquals(list.getColumn(i).value.getValue(), expectedValues[i]); 38 | } 39 | 40 | list.updateValues(null); 41 | for (int i = 0; i < list.size(); i++) { 42 | assertEquals(list.getColumn(i).value.getValue(), expectedValues[i]); 43 | } 44 | 45 | List refs = new ArrayList(); 46 | list.updateValues(refs); 47 | for (int i = 0; i < list.size(); i++) { 48 | assertEquals(list.getColumn(i).value.getValue(), expectedValues[i]); 49 | } 50 | 51 | refs.add(new ColumnDefinition("xcolumn", DataType.Str, true, DEFAULT_LENGTH, DEFAULT_PRECISION, DEFAULT_SCALE, 52 | null, "x", null)); 53 | list.updateValues(refs); 54 | for (int i = 0; i < list.size(); i++) { 55 | assertEquals(list.getColumn(i).value.getValue(), expectedValues[i]); 56 | } 57 | 58 | refs.add(new ColumnDefinition("column2", DataType.Int16, true, DEFAULT_LENGTH, DEFAULT_PRECISION, DEFAULT_SCALE, 59 | null, "22", null)); 60 | list.updateValues(refs); 61 | expectedValues = new Object[] { 0, "22", 0 }; 62 | for (int i = 0; i < list.size(); i++) { 63 | assertEquals(list.getColumn(i).value.getValue(), expectedValues[i]); 64 | } 65 | } 66 | 67 | @Test(groups = { "unit" }) 68 | public void testFromString() { 69 | String inlineSchema = "a Nullable(UInt8) default 3, b Enum8('N/A'=1, 'SB'=2)"; 70 | TableDefinition def = TableDefinition.fromString(inlineSchema); 71 | 72 | assertNotNull(def.getColumn(1).toString()); 73 | } 74 | } -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/impl/JsonFileRepositoryTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.impl; 17 | 18 | import static org.testng.Assert.*; 19 | 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | import java.util.function.Consumer; 23 | 24 | import com.clickhouse.jdbcbridge.core.Extension; 25 | import com.clickhouse.jdbcbridge.core.ExtensionManager; 26 | import com.clickhouse.jdbcbridge.core.NamedDataSource; 27 | import com.clickhouse.jdbcbridge.core.Repository; 28 | import com.clickhouse.jdbcbridge.core.RepositoryManager; 29 | import com.clickhouse.jdbcbridge.core.Utils; 30 | 31 | import org.testng.annotations.Test; 32 | 33 | import io.vertx.core.json.JsonObject; 34 | 35 | public class JsonFileRepositoryTest { 36 | static class TestExtensionManager implements ExtensionManager { 37 | private final RepositoryManager repoManager = new DefaultRepositoryManager(); 38 | 39 | @Override 40 | public Extension getExtension(Class clazz) { 41 | // TODO Auto-generated method stub 42 | return null; 43 | } 44 | 45 | @Override 46 | public RepositoryManager getRepositoryManager() { 47 | return repoManager; 48 | } 49 | 50 | @Override 51 | public void registerConfigLoader(String configPath, Consumer consumer) { 52 | } 53 | 54 | @Override 55 | public Map getScriptableObjects() { 56 | return new HashMap<>(); 57 | } 58 | } 59 | 60 | @Test(groups = { "unit" }) 61 | public void testGet() { 62 | Repository repo = new JsonFileRepository<>(NamedDataSource.class); 63 | assertNull(repo.get("non-existent-ds")); 64 | assertNull(repo.get("invalid-type:non-existent-ds")); 65 | 66 | repo.registerType("jdbc", new Extension(NamedDataSource.class)); 67 | assertThrows(IllegalArgumentException.class, new ThrowingRunnable() { 68 | @Override 69 | public void run() throws Throwable { 70 | repo.get("non-existent-ds"); 71 | } 72 | }); 73 | assertThrows(IllegalArgumentException.class, new ThrowingRunnable() { 74 | @Override 75 | public void run() throws Throwable { 76 | repo.get("invalid-type:non-existent-ds"); 77 | } 78 | }); 79 | assertThrows(IllegalArgumentException.class, new ThrowingRunnable() { 80 | @Override 81 | public void run() throws Throwable { 82 | repo.get("jenkins:https://my.ci-server.org/internal/"); 83 | } 84 | }); 85 | 86 | String uri = "jdbc:mysql://localhost:3306/test?useSSL=false"; 87 | NamedDataSource ds = repo.get(uri); 88 | assertNotNull(ds); 89 | assertEquals(ds.getId(), uri); 90 | 91 | uri = "jdbc:weird:vendor:hostname:1234?database=test"; 92 | ds = repo.get(uri); 93 | assertNotNull(ds); 94 | assertEquals(ds.getId(), uri); 95 | } 96 | 97 | @Test(groups = { "unit" }) 98 | public void testPutAndGet() { 99 | Repository repo = new JsonFileRepository<>(NamedDataSource.class); 100 | 101 | assertThrows(NullPointerException.class, new ThrowingRunnable() { 102 | @Override 103 | public void run() throws Throwable { 104 | repo.put(null, null); 105 | } 106 | }); 107 | assertThrows(NullPointerException.class, new ThrowingRunnable() { 108 | @Override 109 | public void run() throws Throwable { 110 | repo.put("random", null); 111 | } 112 | }); 113 | 114 | String id = "nds"; 115 | JsonObject config = Utils.loadJsonFromFile("src/test/resources/datasources/test-nds.json") 116 | .getJsonObject("test-nds"); 117 | 118 | NamedDataSource nds1 = new NamedDataSource("nds1", repo, config); 119 | repo.put(id, nds1); 120 | assertNull(repo.get(nds1.getId())); 121 | assertEquals(repo.get(id), nds1); 122 | assertEquals(repo.get("nds01"), nds1); 123 | assertEquals(repo.get("nds001"), nds1); 124 | 125 | NamedDataSource nds2 = new NamedDataSource("nds02", repo, config); 126 | repo.put("nds02", nds2); 127 | assertEquals(repo.get(id), nds1); 128 | assertEquals(repo.get("nds02"), nds2); 129 | assertEquals(repo.get("nds01"), nds2); 130 | assertEquals(repo.get("nds001"), nds2); 131 | 132 | NamedDataSource nds3 = new NamedDataSource(id, repo, config); 133 | repo.put(id, nds3); 134 | assertEquals(repo.get(id), nds3); 135 | assertEquals(repo.get("nds02"), nds2); 136 | assertEquals(repo.get("nds01"), nds3); 137 | assertEquals(repo.get("nds001"), nds3); 138 | 139 | NamedDataSource nds4 = new NamedDataSource("nds04", repo, 140 | Utils.loadJsonFromFile("src/test/resources/datasources/test-datasource.json") 141 | .getJsonObject("test-datasource")); 142 | repo.put("nds01", nds4); 143 | assertEquals(repo.get(id), nds3); 144 | assertEquals(repo.get("nds02"), nds2); 145 | assertEquals(repo.get("nds01"), nds4); 146 | assertEquals(repo.get("nds001"), nds3); 147 | 148 | NamedDataSource nds5 = new NamedDataSource(id, repo, config); 149 | repo.put(id, nds5); 150 | repo.put(id, nds4); 151 | assertEquals(repo.get(id), nds4); 152 | assertEquals(repo.get("nds02"), nds2); 153 | assertNull(repo.get("nds01")); 154 | assertNull(repo.get("nds001")); 155 | } 156 | 157 | @Test(groups = { "sit" }) 158 | public void testSrvRecordSupport() { 159 | Repository repo = new JsonFileRepository<>(NamedDataSource.class); 160 | 161 | String host = "_sip._udp.sip.voice.google.com"; 162 | String port = "5060"; 163 | String hostAndPort = host + ":" + port; 164 | 165 | assertEquals(repo.resolve("jdbc://{{ _sip._udp.sip.voice.google.com }}/aaa"), "jdbc://" + hostAndPort + "/aaa"); 166 | } 167 | } -------------------------------------------------------------------------------- /src/test/java/com/clickhouse/jdbcbridge/impl/ScriptDataSourceTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2022, Zhichun Wu 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.clickhouse.jdbcbridge.impl; 17 | 18 | import static org.testng.Assert.*; 19 | 20 | import java.util.List; 21 | 22 | import javax.script.Invocable; 23 | import javax.script.ScriptEngine; 24 | import javax.script.ScriptEngineFactory; 25 | import javax.script.ScriptEngineManager; 26 | 27 | import org.testng.annotations.Test; 28 | 29 | public class ScriptDataSourceTest { 30 | @Test(groups = { "unit" }) 31 | public void testScriptEval() { 32 | ScriptEngineManager manager = new ScriptEngineManager(); 33 | List factories = manager.getEngineFactories(); 34 | assertNotNull(factories); 35 | 36 | ScriptEngine engine = manager.getEngineByExtension("js"); 37 | assertNotNull(engine); 38 | 39 | try { 40 | Object obj = engine.eval("var i = 1"); 41 | obj = engine.eval("1"); 42 | obj = engine.eval("function run() { return [['c1', 'uint8'], ['c2', 'String']] }"); 43 | obj = engine.get("run"); 44 | 45 | Invocable inv = (Invocable) engine; 46 | obj = inv.invokeFunction("run"); 47 | obj = null; 48 | } catch (Exception e) { 49 | e.printStackTrace(); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/test/resources/datasources/test-datasource.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../docker/config/datasource-schema.json", 3 | 4 | "test-datasource": { 5 | "jdbcUrl": "jdbc:mysql://:/?useCompression=true&useOldAliasMetadataBehavior=true", 6 | "dataSource": { 7 | "user": "", 8 | "password": "" 9 | }, 10 | "columns": [ 11 | { 12 | "name": "c_bool", 13 | "type": "Boolean", 14 | "nullable": false, 15 | "value": "1" 16 | }, 17 | { 18 | "name": "c_int8", 19 | "type": "Int8", 20 | "nullable": false, 21 | "value": "2" 22 | }, 23 | { 24 | "name": "c_int16", 25 | "type": "Int16", 26 | "nullable": false, 27 | "value": "2" 28 | }, 29 | { 30 | "name": "c_int32", 31 | "type": "Int32", 32 | "nullable": false, 33 | "value": "2" 34 | }, 35 | { 36 | "name": "c_int64", 37 | "type": "Int64", 38 | "nullable": false, 39 | "value": "2" 40 | }, 41 | { 42 | "name": "c_uint8", 43 | "type": "UInt8", 44 | "nullable": false, 45 | "value": "2" 46 | }, 47 | { 48 | "name": "c_uint16", 49 | "type": "UInt16", 50 | "nullable": false, 51 | "value": "2" 52 | }, 53 | { 54 | "name": "c_uint32", 55 | "type": "UInt32", 56 | "nullable": false, 57 | "value": "2" 58 | }, 59 | { 60 | "name": "c_uint64", 61 | "type": "UInt64", 62 | "nullable": false, 63 | "value": "2" 64 | }, 65 | { 66 | "name": "c_float32", 67 | "type": "Float32", 68 | "nullable": false, 69 | "value": "2.0" 70 | }, 71 | { 72 | "name": "c_float64", 73 | "type": "Float64", 74 | "nullable": false, 75 | "value": "2.0" 76 | }, 77 | { 78 | "name": "c_date", 79 | "type": "Date", 80 | "nullable": false, 81 | "value": "2" 82 | }, 83 | { 84 | "name": "c_datetime", 85 | "type": "DateTime", 86 | "nullable": false, 87 | "value": "2" 88 | }, 89 | { 90 | "name": "c_datetime64", 91 | "type": "DateTime64", 92 | "nullable": false, 93 | "value": "2" 94 | }, 95 | { 96 | "name": "c_decimal", 97 | "type": "Decimal", 98 | "nullable": false, 99 | "value": "2.0" 100 | }, 101 | { 102 | "name": "c_decimal32", 103 | "type": "Decimal32", 104 | "nullable": false, 105 | "value": "2.0" 106 | }, 107 | { 108 | "name": "c_decimal64", 109 | "type": "Decimal64", 110 | "nullable": false, 111 | "value": "2.0" 112 | }, 113 | { 114 | "name": "c_decimal128", 115 | "type": "Decimal128", 116 | "nullable": false, 117 | "value": "2.0" 118 | }, 119 | { 120 | "name": "c_decimal256", 121 | "type": "Decimal256", 122 | "nullable": false, 123 | "value": "2.0" 124 | }, 125 | { 126 | "name": "c_enum", 127 | "type": "Enum", 128 | "nullable": false, 129 | "value": "2", 130 | "options": { 131 | "A": 1, 132 | "B": 2 133 | } 134 | }, 135 | { 136 | "name": "c_enum8", 137 | "type": "Enum8", 138 | "nullable": false, 139 | "value": "2" 140 | }, 141 | { 142 | "name": "c_enum16", 143 | "type": "Enum16", 144 | "nullable": false, 145 | "value": "2" 146 | }, 147 | { 148 | "name": "c_ipv4", 149 | "type": "IPv4", 150 | "nullable": false, 151 | "value": "2" 152 | }, 153 | { 154 | "name": "c_ipv6", 155 | "type": "IPv6", 156 | "nullable": false, 157 | "value": "2" 158 | }, 159 | { 160 | "name": "c_fixedstr", 161 | "type": "FixedString", 162 | "length": 3, 163 | "nullable": false, 164 | "value": "2" 165 | }, 166 | { 167 | "name": "c_str", 168 | "type": "String", 169 | "nullable": false, 170 | "value": "2" 171 | }, 172 | { 173 | "name": "c_uuid", 174 | "type": "UUID", 175 | "nullable": false, 176 | "value": "2" 177 | } 178 | ], 179 | "defaults": { 180 | "Boolean": 0, 181 | "Int8": 3, 182 | "Int16": 3, 183 | "Int32": 3, 184 | "Int64": 3, 185 | "Int128": 3, 186 | "Int256": 3, 187 | "UInt8": 3, 188 | "UInt16": 3, 189 | "UInt32": 3, 190 | "UInt64": 3, 191 | "UInt128": 3, 192 | "UInt256": 3, 193 | "Float32": 3.0, 194 | "Float64": 3.0, 195 | "Date": 3, 196 | "DateTime": 3, 197 | "DateTime64": 3.0, 198 | "Decimal": 3.0, 199 | "Decimal32": 3.0, 200 | "Decimal64": 3.0, 201 | "Decimal128": 3.0, 202 | "Decimal256": 3.0, 203 | "Enum": 3, 204 | "Enum8": 3, 205 | "Enum16": 3, 206 | "IPv4": 3, 207 | "IPv6": "3", 208 | "FixedString": "3", 209 | "String": "3", 210 | "UUID": "3" 211 | } 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /src/test/resources/datasources/test-legacy.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../docker/config/datasource-schema.json", 3 | 4 | "test-legacy": { 5 | "jdbcUrl": "jdbc:hive2:///;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=", 6 | "driverClassName": "org.apache.hive.jdbc.HiveDriver", 7 | "username": "", 8 | "password": "", 9 | "parameters": { 10 | "fetch_size": 1000 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/test/resources/datasources/test-nds.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "../../../../docker/config/datasource-schema.json", 3 | 4 | "test-nds": { 5 | "aliases": ["nds01", "nds001"] 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/test/resources/logging.properties: -------------------------------------------------------------------------------- 1 | handlers=java.util.logging.ConsoleHandler 2 | 3 | java.util.logging.ConsoleHandler.level=DEBUG 4 | java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter 5 | java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$-7s] %5$s %n 6 | 7 | .level=DEBUG 8 | -------------------------------------------------------------------------------- /src/test/resources/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestTimeout": 5000, 3 | "queryTimeout": 30000, 4 | "configScanPeriod": 5000, 5 | "repositories": [ 6 | { 7 | "entity": "com.clickhouse.jdbcbridge.core.NamedDataSource", 8 | "repository": "com.clickhouse.jdbcbridge.impl.JsonFileRepository" 9 | }, 10 | { 11 | "entity": "com.clickhouse.jdbcbridge.core.NamedSchema", 12 | "repository": "com.clickhouse.jdbcbridge.JdbcBridgeVerticleTest$TestRepository" 13 | } 14 | ], 15 | "extensions": [ 16 | { 17 | "class": "com.clickhouse.jdbcbridge.impl.JdbcDataSource" 18 | }, 19 | { 20 | "class": "com.clickhouse.jdbcbridge.impl.ConfigDataSource" 21 | }, 22 | { 23 | "libUrls": ["extensions/script-engines"], 24 | "class": "com.clickhouse.jdbcbridge.impl.ScriptDataSource" 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /src/test/resources/simple.query: -------------------------------------------------------------------------------- 1 | select 1 as a, 2 | 2 as b -------------------------------------------------------------------------------- /src/test/resources/sit/ch-server/jdbc-bridge.xml: -------------------------------------------------------------------------------- 1 | 2 | 0.0.0.0 3 | 4 | 5 | jdbc_bridge_server 6 | 9019 7 | 8 | 9 | -------------------------------------------------------------------------------- /src/test/resources/sit/jdbc-bridge/config/datasources/mariadb.json: -------------------------------------------------------------------------------- 1 | { 2 | "mariadb": { 3 | "driverClassName": "org.mariadb.jdbc.Driver", 4 | "jdbcUrl": "jdbc:mariadb://mariadb_server:3306/test?useSSL=false&useCompression=false", 5 | "dataSource": { 6 | "user": "root", 7 | "password": "root" 8 | }, 9 | "initializationFailTimeout": 0, 10 | "minimumIdle": 0, 11 | "maximumPoolSize": 5 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/test/resources/sit/jdbc-bridge/config/datasources/postgresql.json: -------------------------------------------------------------------------------- 1 | { 2 | "postgresql": { 3 | "converter": { 4 | "mappings": [{ "nativeType": "bool", "to": "String" }] 5 | }, 6 | "driverClassName": "org.postgresql.Driver", 7 | "jdbcUrl": "jdbc:postgresql://postgresql_server:5432/postgres", 8 | "username": "sa", 9 | "password": "sa", 10 | "initializationFailTimeout": 0, 11 | "minimumIdle": 0, 12 | "maximumPoolSize": 5 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/test/resources/test.json: -------------------------------------------------------------------------------- 1 | { "b": { "x": "y" }, "a": 1 } 2 | --------------------------------------------------------------------------------