├── .editorconfig ├── .github ├── dependabot.yml ├── renovate.json └── workflows │ ├── backport.yml │ └── ci-pipeline.yml ├── .gitignore ├── CODE_OF_CONDUCT.MD ├── COPYING-HEADER.txt ├── LICENSE ├── README.md ├── docker ├── application.yaml ├── clickhouse-config.xml ├── clickhouse-users.xml └── docker-compose.yml ├── how-it-works.jpg ├── pom.xml ├── renovate.json └── src ├── main └── java │ └── io │ └── zeebe │ └── clickhouse │ └── exporter │ ├── ClickHouseExporter.java │ ├── ClickHouseExporterClient.java │ ├── ExporterConfiguration.java │ └── importer │ ├── ClickHouseConfig.java │ ├── ErrorImporter.java │ ├── IncidentImporter.java │ ├── JobImporter.java │ ├── MessageImporter.java │ ├── MessageSubscriptionImporter.java │ ├── ProcessImporter.java │ ├── ProcessInstanceImporter.java │ ├── SignalSubscriptionImporter.java │ ├── TimerImporter.java │ └── VariableImporter.java └── test ├── java └── io │ └── zeebe │ └── clickhouse │ └── exporter │ └── ClickHouseExporterIT.java └── resources ├── application.yaml └── logback-test.xml /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | indent_style = space 3 | charset = utf-8 4 | end_of_line = lf 5 | trim_trailing_whitespace = true 6 | insert_final_newline = true 7 | 8 | [*.xml] 9 | indent_size = 2 10 | 11 | [*.html] 12 | indent_size = 2 13 | 14 | [*.md] 15 | indent_size = 2 16 | 17 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "maven" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | 13 | - package-ecosystem: "github-actions" 14 | directory: "/" 15 | schedule: 16 | interval: "daily" 17 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { "enabled": false } 2 | -------------------------------------------------------------------------------- /.github/workflows/backport.yml: -------------------------------------------------------------------------------- 1 | name: Backport labeled merged pull requests 2 | on: 3 | pull_request: 4 | types: [closed] 5 | issue_comment: 6 | types: [created] 7 | jobs: 8 | build: 9 | name: Create backport PRs 10 | runs-on: ubuntu-latest 11 | # Only run when pull request is merged 12 | # or when a comment containing `/backport` is created 13 | if: > 14 | ( 15 | github.event_name == 'pull_request' && 16 | github.event.pull_request.merged 17 | ) || ( 18 | github.event_name == 'issue_comment' && 19 | github.event.issue.pull_request && 20 | contains(github.event.comment.body, '/backport') 21 | ) 22 | steps: 23 | - uses: actions/checkout@v4 24 | - uses: korthout/backport-action@v3 25 | -------------------------------------------------------------------------------- /.github/workflows/ci-pipeline.yml: -------------------------------------------------------------------------------- 1 | # If this workflow is triggered by a push to $default_branch, it 2 | # deploys a SNAPSHOT 3 | # If this workflow is triggered by publishing a Release, it 4 | # deploys a RELEASE with the selected version 5 | # updates the project version by incrementing the patch version 6 | # commits the version update change to the repository's default branch ($default_branch). 7 | name: Build, test and deploy artifacts with Maven 8 | on: 9 | pull_request: { } 10 | push: { } 11 | workflow_dispatch: { } 12 | release: 13 | types: [ published ] 14 | jobs: 15 | build: 16 | runs-on: ubuntu-24.04 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Cache 20 | uses: actions/cache@v4.1.2 21 | with: 22 | path: ~/.m2/repository 23 | key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} 24 | restore-keys: | 25 | ${{ runner.os }}-maven- 26 | - name: Set up Java environment 27 | uses: actions/setup-java@v4 28 | with: 29 | java-version: 11 30 | distribution: temurin 31 | gpg-private-key: ${{ secrets.MAVEN_CENTRAL_GPG_SIGNING_KEY_SEC }} 32 | gpg-passphrase: MAVEN_CENTRAL_GPG_PASSPHRASE 33 | - name: Build 34 | id: build 35 | run: mvn -B -U -Dsurefire.rerunFailingTestsCount=5 clean install 36 | - name: Archive Test Results on Failure 37 | uses: actions/upload-artifact@v4 38 | if: failure() 39 | with: 40 | name: test-results 41 | path: target/surefire-reports/ 42 | retention-days: 7 43 | - name: Publish Unit Test Results 44 | id: publish 45 | uses: EnricoMi/publish-unit-test-result-action@v2 46 | if: failure() 47 | with: 48 | files: target/surefire-reports/*.xml 49 | - if: github.event.release || github.event_name == 'workflow_dispatch' 50 | name: Deploy SNAPSHOT / Release 51 | uses: camunda-community-hub/community-action-maven-release@v1.2.2 52 | with: 53 | release-version: ${{ github.event.release.tag_name }} 54 | release-profile: community-action-maven-release 55 | nexus-usr: ${{ secrets.NEXUS_USR }} 56 | nexus-psw: ${{ secrets.NEXUS_PSW }} 57 | maven-usr: ${{ secrets.MAVEN_CENTRAL_DEPLOYMENT_USR }} 58 | maven-psw: ${{ secrets.MAVEN_CENTRAL_DEPLOYMENT_PSW }} 59 | maven-gpg-passphrase: ${{ secrets.MAVEN_CENTRAL_GPG_SIGNING_KEY_PASSPHRASE }} 60 | github-token: ${{ secrets.GITHUB_TOKEN }} 61 | id: release 62 | - if: github.event.release 63 | name: Attach artifacts to GitHub Release (Release only) 64 | uses: actions/upload-release-asset@v1 65 | env: 66 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 67 | with: 68 | upload_url: ${{ github.event.release.upload_url }} 69 | asset_path: ${{ steps.release.outputs.artifacts_archive_path }} 70 | asset_name: ${{ steps.release.outputs.artifacts_archive_path }} 71 | asset_content_type: application/zip 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | pom.xml.tag 3 | pom.xml.releaseBackup 4 | pom.xml.versionsBackup 5 | pom.xml.next 6 | release.properties 7 | dependency-reduced-pom.xml 8 | buildNumber.properties 9 | .mvn/timing.properties 10 | # https://maven.apache.org/wrapper/#usage-without-binary-jar 11 | .mvn/wrapper/maven-wrapper.jar 12 | /.classpath 13 | /.project 14 | /.settings/ 15 | /give-me-a-name.iml 16 | /.idea/ 17 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.MD: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ### View the [Camunda Code of Conduct](https://camunda.com/events/code-conduct/) and find ways to report violations. 4 | -------------------------------------------------------------------------------- /COPYING-HEADER.txt: -------------------------------------------------------------------------------- 1 | Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 2 | one or more contributor license agreements. See the NOTICE file distributed 3 | with this work for additional information regarding copyright ownership. 4 | Licensed under the Zeebe Community License 1.1. You may not use this file 5 | except in compliance with the Zeebe Community License 1.1. 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Community extension badge](https://img.shields.io/badge/Community%20Extension-An%20open%20source%20community%20maintained%20project-FF4700)](https://github.com/camunda-community-hub/community) 2 | [![Community badge: Incubating](https://img.shields.io/badge/Lifecycle-Incubating-blue)](https://github.com/Camunda-Community-Hub/community/blob/main/extension-lifecycle.md#incubating-) 3 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 4 | 5 | [![Compatible with: Camunda Platform 8](https://img.shields.io/badge/Compatible%20with-Camunda%20Platform%208-0072Ce)](https://github.com/camunda-community-hub/community/blob/main/extension-lifecycle.md#compatiblilty) 6 | 7 | # Zeebe ClickHouse Exporter 8 | 9 | [Exporters](https://docs.camunda.io/docs/next/components/zeebe/technical-concepts/architecture/#exporters) allow you to tap into the Zeebe event stream on a partition and export selected events to other systems. You can filter events, perform transformations, and even trigger side-effects from an exporter. 10 | 11 | Read a two-part series about building Zeebe exporters on the Zeebe blog: [Part One](https://camunda.com/blog/2019/05/exporter-part-1/) | [Part Two](https://camunda.com/blog/2019/06/exporter-part-2/). 12 | 13 | This [Zeebe ClickHouse Exporter](https://github.com/camunda-community-hub/zeebe-clickhouse-exporter) is build on [Zeebe Exporter Demo](https://github.com/jwulf/zeebe-exporter-demo.git) and [ClickHouse Java Libraries](https://github.com/ClickHouse/clickhouse-java.git). 14 | 15 | You can export records from Zeebe to ClickHouse and query data from ClickHouse use you own program language. 16 | 17 | ![How it works](how-it-works.jpg) 18 | 19 | 20 | ## About ClickHouse 21 | 22 | [ClickHouse](https://clickhouse.com/clickhouse) is a column-oriented database that enables its users to generate powerful analytics, using SQL queries, in real-time. 23 | 24 | ClickHouse is an Open Source OLAP database management system. 25 | 26 | ClickHouse runs on ClickHouse Cloud or any Linux, FreeBSD, or macOS system with x86_64, AArch64, or PowerPC64LE CPU architecture. 27 | 28 | [ClickHouse Quick Start](https://clickhouse.com/docs/en/getting-started/quick-start/) follow these steps to get up and running with ClickHouse. 29 | 30 | [ClickHouse Java Libraries](https://github.com/ClickHouse/clickhouse-java) for connecting to ClickHouse and processing data in various formats. Java client is async, lightweight, and low-overhead library for ClickHouse 31 | 32 | [ClickHouse Asynchronous Inserts](https://clickhouse.com/docs/en/cloud/bestpractices/asynchronous-inserts) inserting data into ClickHouse in large batches is a best practice. 33 | 34 | ## Deploy the Exporter 35 | 36 | 1. Build the exporter, using `mvn package`. 37 | 2. Copy the resulting `zeebe-clickhouse-exporter-1.0-SNAPSHOT.jar` file to the `exporters` directory of your Zeebe broker. 38 | 3. Edit the `application.yaml` file, and add an entry for the exporter, you can change 39 | 40 | * the ClickHouse jdbc url `chUrl` 41 | * the ClickHouse user `chUser` 42 | * the ClickHouse password `chPassword` 43 | 44 | ``` 45 | exporters: 46 | clickhouse: 47 | className: io.zeebe.clickhouse.exporter.ClickHouseExporter 48 | jarPath: exporters/zeebe-clickhouse-exporter-1.0-SNAPSHOT.jar 49 | args: 50 | chUrl: jdbc:ch://127.0.0.1:8123/default 51 | chUser: default 52 | chPassword: clickhouse123 53 | ``` 54 | The values can be overridden by environment variables. 55 | 56 | * set `chUrl` with `ZEEBE_CLICKHOUSE_URL` (e.g. `export ZEEBE_CLICKHOUSE_URL=jdbc:ch://127.0.0.1:8123/default`) 57 | * set `chUser` with `ZEEBE_CLICKHOUSE_USER` (e.g. `export ZEEBE_CLICKHOUSE_USER=default`) 58 | * set `chPassword` with `ZEEBE_CLICKHOUSE_PASSWORD` (e.g. `export ZEEBE_CLICKHOUSE_PASSWORD=""`) 59 | 60 | 61 | ## ClickHouse Zeebe Tables 62 | 63 | ``` 64 | chk :) show tables; 65 | 66 | SHOW TABLES 67 | 68 | Query id: e18a6412-c726-48d2-b73b-6e78c0af3b12 69 | 70 | ┌─name─────────────────┐ 71 | │ CLICKHOUSE_CONFIG │ 72 | │ ELEMENT_INSTANCE │ 73 | │ ERROR │ 74 | │ INCIDENT │ 75 | │ JOB │ 76 | │ MESSAGE │ 77 | │ MESSAGE_SUBSCRIPTION │ 78 | │ PROCESS │ 79 | │ PROCESS_INSTANCE │ 80 | │ TIMER │ 81 | │ VARIABLE │ 82 | └──────────────────────┘ 83 | 84 | 11 rows in set. Elapsed: 0.002 sec. 85 | 86 | chk :) 87 | ``` 88 | 89 | ## ClickHouse Query Process Records 90 | ``` 91 | chk :) select KEY_,BPMN_PROCESS_ID_,RESOURCE_NAME_,TIMESTAMP_,PARTITION_ID_,POSITION_,VERSION_ from PROCESS; 92 | 93 | SELECT 94 | KEY_, 95 | BPMN_PROCESS_ID_, 96 | RESOURCE_NAME_, 97 | TIMESTAMP_, 98 | PARTITION_ID_, 99 | POSITION_, 100 | VERSION_ 101 | FROM PROCESS 102 | 103 | Query id: 3d27fd6a-87fb-4891-b27d-8b445ac7bc64 104 | 105 | ┌─────────────KEY_─┬─BPMN_PROCESS_ID_─┬─RESOURCE_NAME_───────┬──────────────TIMESTAMP_─┬─PARTITION_ID_─┬─POSITION_─┬─VERSION_─┐ 106 | │ 2251799813685986 │ Process_0k1nnh5 │ Process_0k1nnh5.bpmn │ 2023-03-11 12:56:00.614 │ 1 │ 1900 │ 4 │ 107 | │ 2251799813686688 │ Process_0k1nnh5 │ Process_0k1nnh5.bpmn │ 2023-03-11 13:22:36.327 │ 1 │ 3704 │ 5 │ 108 | │ 2251799813687390 │ Process_0k1nnh5 │ Process_0k1nnh5.bpmn │ 2023-03-11 13:45:48.440 │ 1 │ 5508 │ 6 │ 109 | │ 2251799813687393 │ Process_0k1nnh5 │ Process_0k1nnh5.bpmn │ 2023-03-11 13:49:07.744 │ 1 │ 5513 │ 7 │ 110 | │ 2251799813687395 │ Process_0k1nnh5 │ Process_0k1nnh5.bpmn │ 2023-03-11 13:50:22.217 │ 1 │ 5518 │ 8 │ 111 | └──────────────────┴──────────────────┴──────────────────────┴─────────────────────────┴───────────────┴───────────┴──────────┘ 112 | 113 | 5 rows in set. Elapsed: 0.003 sec. 114 | 115 | chk :) 116 | ``` 117 | 118 | ## ClickHouse Query Process Instance Records 119 | ``` 120 | chk :) select * from PROCESS_INSTANCE limit 10; 121 | 122 | SELECT * 123 | FROM PROCESS_INSTANCE 124 | LIMIT 10 125 | 126 | Query id: 790c13cc-5066-49c0-9068-4e6e610b8cf0 127 | 128 | ┌─────────────KEY_─┬─BPMN_PROCESS_ID_─┬─PROCESS_DEFINITION_KEY_─┬──────────────────START_─┬─END_─┬─PARTITION_ID_─┬─VERSION_─┬─STATE_─┬─PARENT_PROCESS_INSTANCE_KEY_─┬─PARENT_ELEMENT_INSTANCE_KEY_─┐ 129 | │ 2251799813685989 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:03.649 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 130 | │ 2251799813685996 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:06.651 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 131 | │ 2251799813686003 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:09.632 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 132 | │ 2251799813686010 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:12.639 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 133 | │ 2251799813686017 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:15.643 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 134 | │ 2251799813686024 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:18.623 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 135 | │ 2251799813686031 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:21.621 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 136 | │ 2251799813686038 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:24.651 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 137 | │ 2251799813686045 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:27.649 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 138 | │ 2251799813686052 │ Process_0k1nnh5 │ 2251799813685986 │ 2023-03-11 12:56:30.649 │ ᴺᵁᴸᴸ │ 1 │ 4 │ Active │ -1 │ -1 │ 139 | └──────────────────┴──────────────────┴─────────────────────────┴─────────────────────────┴──────┴───────────────┴──────────┴────────┴──────────────────────────────┴──────────────────────────────┘ 140 | 141 | 10 rows in set. Elapsed: 0.003 sec. 142 | 143 | chk :) 144 | ``` 145 | 146 | ## ClickHouse Query Element Instance Records 147 | ``` 148 | chk :) select * from ELEMENT_INSTANCE where PROCESS_INSTANCE_KEY_=2251799813689134 order by TIMESTAMP_ asc; 149 | 150 | SELECT * 151 | FROM ELEMENT_INSTANCE 152 | WHERE PROCESS_INSTANCE_KEY_ = 2251799813689134 153 | ORDER BY TIMESTAMP_ ASC 154 | 155 | Query id: ebbffa4f-c188-47b1-99a4-3d18e84645bb 156 | 157 | ┌─ID──────┬─────────────KEY_─┬─BPMN_PROCESS_ID_─┬─PROCESS_DEFINITION_KEY_─┬──────────────TIMESTAMP_─┬─INTENT_─────────────┬─PARTITION_ID_─┬─POSITION_─┬─PROCESS_INSTANCE_KEY_─┬──FLOW_SCOPE_KEY_─┬─ELEMENT_ID_──────┬─BPMN_ELEMENT_TYPE_─┐ 158 | │ 1-9991 │ 2251799813689134 │ Process_1udqk7z │ 2251799813685323 │ 2023-03-12 11:29:13.349 │ ELEMENT_ACTIVATING │ 1 │ 9991 │ 2251799813689134 │ -1 │ Process_1udqk7z │ PROCESS │ 159 | │ 1-9992 │ 2251799813689134 │ Process_1udqk7z │ 2251799813685323 │ 2023-03-12 11:29:13.349 │ ELEMENT_ACTIVATED │ 1 │ 9992 │ 2251799813689134 │ -1 │ Process_1udqk7z │ PROCESS │ 160 | │ 1-9994 │ 2251799813689137 │ Process_1udqk7z │ 2251799813685323 │ 2023-03-12 11:29:13.349 │ ELEMENT_ACTIVATING │ 1 │ 9994 │ 2251799813689134 │ 2251799813689134 │ StartEvent_1 │ START_EVENT │ 161 | │ 1-9995 │ 2251799813689137 │ Process_1udqk7z │ 2251799813685323 │ 2023-03-12 11:29:13.349 │ ELEMENT_ACTIVATED │ 1 │ 9995 │ 2251799813689134 │ 2251799813689134 │ StartEvent_1 │ START_EVENT │ 162 | │ 1-9997 │ 2251799813689137 │ Process_1udqk7z │ 2251799813685323 │ 2023-03-12 11:29:13.352 │ ELEMENT_COMPLETING │ 1 │ 9997 │ 2251799813689134 │ 2251799813689134 │ StartEvent_1 │ START_EVENT │ 163 | │ 1-9998 │ 2251799813689137 │ Process_1udqk7z │ 2251799813685323 │ 2023-03-12 11:29:13.352 │ ELEMENT_COMPLETED │ 1 │ 9998 │ 2251799813689134 │ 2251799813689134 │ StartEvent_1 │ START_EVENT │ 164 | │ 1-9999 │ 2251799813689138 │ Process_1udqk7z │ 2251799813685323 │ 2023-03-12 11:29:13.352 │ SEQUENCE_FLOW_TAKEN │ 1 │ 9999 │ 2251799813689134 │ 2251799813689134 │ Flow_0j9gwbw │ SEQUENCE_FLOW │ 165 | │ 1-10001 │ 2251799813689139 │ Process_1udqk7z │ 2251799813685323 │ 2023-03-12 11:29:13.356 │ ELEMENT_ACTIVATING │ 1 │ 10001 │ 2251799813689134 │ 2251799813689134 │ Activity_0v39lo1 │ USER_TASK │ 166 | │ 1-10003 │ 2251799813689139 │ Process_1udqk7z │ 2251799813685323 │ 2023-03-12 11:29:13.356 │ ELEMENT_ACTIVATED │ 1 │ 10003 │ 2251799813689134 │ 2251799813689134 │ Activity_0v39lo1 │ USER_TASK │ 167 | └─────────┴──────────────────┴──────────────────┴─────────────────────────┴─────────────────────────┴─────────────────────┴───────────────┴───────────┴───────────────────────┴──────────────────┴──────────────────┴────────────────────┘ 168 | 169 | 9 rows in set. Elapsed: 0.044 sec. Processed 9.09 thousand rows, 1.43 MB (204.70 thousand rows/s., 32.24 MB/s.) 170 | 171 | chk :) 172 | ``` 173 | 174 | ## ClickHouse Query Timer Records 175 | ``` 176 | chk :) select * from TIMER limit 10; 177 | 178 | SELECT * 179 | FROM TIMER 180 | LIMIT 10 181 | 182 | Query id: 96d218f8-7628-4a98-978a-d018bc420b8f 183 | 184 | ┌─────────────KEY_─┬─STATE_────┬─REPETITIONS─┬──────────────TIMESTAMP_─┬───────────────DUE_DATE_─┬─PROCESS_INSTANCE_KEY_─┬─PROCESS_DEFINITION_KEY_─┬─ELEMENT_INSTANCE_KEY_─┬─TARGET_ELEMENT_ID_─┐ 185 | │ 2251799813685987 │ triggered │ 100 │ 2023-03-11 12:56:03.649 │ 2023-03-11 12:56:03.614 │ 2251799813685989 │ 2251799813685986 │ -1 │ StartEvent_1 │ 186 | │ 2251799813685991 │ triggered │ 99 │ 2023-03-11 12:56:06.651 │ 2023-03-11 12:56:06.614 │ 2251799813685996 │ 2251799813685986 │ -1 │ StartEvent_1 │ 187 | │ 2251799813685998 │ triggered │ 98 │ 2023-03-11 12:56:09.632 │ 2023-03-11 12:56:09.614 │ 2251799813686003 │ 2251799813685986 │ -1 │ StartEvent_1 │ 188 | │ 2251799813686005 │ triggered │ 97 │ 2023-03-11 12:56:12.639 │ 2023-03-11 12:56:12.614 │ 2251799813686010 │ 2251799813685986 │ -1 │ StartEvent_1 │ 189 | │ 2251799813686012 │ triggered │ 96 │ 2023-03-11 12:56:15.643 │ 2023-03-11 12:56:15.614 │ 2251799813686017 │ 2251799813685986 │ -1 │ StartEvent_1 │ 190 | │ 2251799813686019 │ triggered │ 95 │ 2023-03-11 12:56:18.623 │ 2023-03-11 12:56:18.614 │ 2251799813686024 │ 2251799813685986 │ -1 │ StartEvent_1 │ 191 | │ 2251799813686026 │ triggered │ 94 │ 2023-03-11 12:56:21.621 │ 2023-03-11 12:56:21.614 │ 2251799813686031 │ 2251799813685986 │ -1 │ StartEvent_1 │ 192 | │ 2251799813686033 │ triggered │ 93 │ 2023-03-11 12:56:24.651 │ 2023-03-11 12:56:24.614 │ 2251799813686038 │ 2251799813685986 │ -1 │ StartEvent_1 │ 193 | │ 2251799813686040 │ triggered │ 92 │ 2023-03-11 12:56:27.649 │ 2023-03-11 12:56:27.614 │ 2251799813686045 │ 2251799813685986 │ -1 │ StartEvent_1 │ 194 | │ 2251799813686047 │ triggered │ 91 │ 2023-03-11 12:56:30.649 │ 2023-03-11 12:56:30.614 │ 2251799813686052 │ 2251799813685986 │ -1 │ StartEvent_1 │ 195 | └──────────────────┴───────────┴─────────────┴─────────────────────────┴─────────────────────────┴───────────────────────┴─────────────────────────┴───────────────────────┴────────────────────┘ 196 | 197 | 10 rows in set. Elapsed: 0.003 sec. 198 | 199 | chk :) 200 | ``` 201 | ## ClickHouse Query Job Records 202 | ``` 203 | chk :) select * from JOB limit 10; 204 | 205 | SELECT * 206 | FROM JOB 207 | LIMIT 10 208 | 209 | Query id: b123ba7f-b552-4dba-9d66-9d30a9478a6d 210 | 211 | ┌─────────────KEY_─┬─BPMN_PROCESS_ID_─┬─ELEMENT_ID_──────┬─WORKER_─┬─JOB_TYPE_─────────────────┬─STATE_──┬─RETRIES_─┬──────────────────START_─┬─END_─┬─PROCESS_INSTANCE_KEY_─┬─ELEMENT_INSTANCE_KEY_─┬─PROCESS_DEFINITION_KEY_─┐ 212 | │ 2251799813685995 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:03.649 │ ᴺᵁᴸᴸ │ 2251799813685989 │ 2251799813685994 │ 2251799813685986 │ 213 | │ 2251799813686002 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:06.651 │ ᴺᵁᴸᴸ │ 2251799813685996 │ 2251799813686001 │ 2251799813685986 │ 214 | │ 2251799813686009 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:09.632 │ ᴺᵁᴸᴸ │ 2251799813686003 │ 2251799813686008 │ 2251799813685986 │ 215 | │ 2251799813686016 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:12.639 │ ᴺᵁᴸᴸ │ 2251799813686010 │ 2251799813686015 │ 2251799813685986 │ 216 | │ 2251799813686023 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:15.643 │ ᴺᵁᴸᴸ │ 2251799813686017 │ 2251799813686022 │ 2251799813685986 │ 217 | │ 2251799813686030 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:18.623 │ ᴺᵁᴸᴸ │ 2251799813686024 │ 2251799813686029 │ 2251799813685986 │ 218 | │ 2251799813686037 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:21.621 │ ᴺᵁᴸᴸ │ 2251799813686031 │ 2251799813686036 │ 2251799813685986 │ 219 | │ 2251799813686044 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:24.651 │ ᴺᵁᴸᴸ │ 2251799813686038 │ 2251799813686043 │ 2251799813685986 │ 220 | │ 2251799813686051 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:27.649 │ ᴺᵁᴸᴸ │ 2251799813686045 │ 2251799813686050 │ 2251799813685986 │ 221 | │ 2251799813686058 │ Process_0k1nnh5 │ Activity_0fhkw5l │ │ io.camunda.zeebe:userTask │ created │ 1 │ 2023-03-11 12:56:30.649 │ ᴺᵁᴸᴸ │ 2251799813686052 │ 2251799813686057 │ 2251799813685986 │ 222 | └──────────────────┴──────────────────┴──────────────────┴─────────┴───────────────────────────┴─────────┴──────────┴─────────────────────────┴──────┴───────────────────────┴───────────────────────┴─────────────────────────┘ 223 | 224 | 10 rows in set. Elapsed: 0.002 sec. 225 | 226 | chk :) 227 | ``` 228 | 229 | ## ClickHouse Query Message Subscription Records 230 | ``` 231 | chk :) select * from MESSAGE_SUBSCRIPTION order by TIMESTAMP_ asc; 232 | 233 | SELECT * 234 | FROM MESSAGE_SUBSCRIPTION 235 | ORDER BY TIMESTAMP_ ASC 236 | 237 | Query id: 3da1ad48-3023-4e58-a17b-a671a87e7228 238 | 239 | ┌─ID_──────────────────────────────────┬─MESSAGE_NAME_───┬─MESSAGE_KEY_─┬──────────────TIMESTAMP_─┬─STATE_──┬─PROCESS_INSTANCE_KEY_─┬─ELEMENT_INSTANCE_KEY_─┬─PROCESS_DEFINITION_KEY_─┬─CORRELATION_KEY_─┬─TARGET_FLOW_NODE_ID_─┐ 240 | │ afd4a50b-0053-4e19-a5f5-74aac0013b04 │ Message_0l9728h │ -1 │ 2023-03-12 04:01:18.081 │ created │ -1 │ -1 │ 2251799813687398 │ │ StartEvent_1 │ 241 | │ e77d6a70-e93c-4521-88aa-3d1e4de46a95 │ Message_0l9728h │ -1 │ 2023-03-12 04:01:33.286 │ deleted │ -1 │ -1 │ 2251799813687398 │ │ StartEvent_1 │ 242 | │ fb7e88a0-79e3-42bd-8a30-d0bf9a887b6e │ Message_0l9728h │ -1 │ 2023-03-12 04:01:33.286 │ created │ -1 │ -1 │ 2251799813687401 │ │ StartEvent_1 │ 243 | └──────────────────────────────────────┴─────────────────┴──────────────┴─────────────────────────┴─────────┴───────────────────────┴───────────────────────┴─────────────────────────┴──────────────────┴──────────────────────┘ 244 | 245 | 3 rows in set. Elapsed: 0.005 sec. 246 | 247 | chk :) 248 | 249 | ``` 250 | 251 | ## ClickHouse Query Message Records 252 | ``` 253 | chk :) select * from MESSAGE order by TIMESTAMP_ asc; 254 | 255 | SELECT * 256 | FROM MESSAGE 257 | ORDER BY TIMESTAMP_ ASC 258 | 259 | Query id: 4263bdf6-585e-4209-ab46-f25a2b0820d1 260 | 261 | ┌─────────────KEY_─┬─NAME_───────────┬──────────────TIMESTAMP_─┬─STATE_────┬─CORRELATION_KEY_─┬─MESSAGE_ID_─┬─PAYLOAD_────────┐ 262 | │ 2251799813687404 │ Message_0l9728h │ 2023-03-12 04:14:37.390 │ published │ 123 │ │ {x=1, y=2} │ 263 | │ 2251799813687415 │ Message_0l9728h │ 2023-03-12 04:15:39.281 │ published │ 9876467 │ │ {x=1, y=2} │ 264 | │ 2251799813687426 │ Message_0l9728h │ 2023-03-12 04:17:54.630 │ published │ 23223423 │ │ {x=11, y=29} │ 265 | │ 2251799813687437 │ Message_0l9728h │ 2023-03-12 04:21:17.489 │ published │ 23233 │ │ {x=111, y=39} │ 266 | │ 2251799813687448 │ Message_0l9728h │ 2023-03-12 04:22:19.181 │ published │ 238868823 │ │ {x=1121, y=339} │ 267 | └──────────────────┴─────────────────┴─────────────────────────┴───────────┴──────────────────┴─────────────┴─────────────────┘ 268 | 269 | 5 rows in set. Elapsed: 0.006 sec. 270 | 271 | chk :) 272 | 273 | ``` 274 | 275 | ## ClickHouse Query Variable Records 276 | ``` 277 | chk :) select * from VARIABLE limit 5; 278 | 279 | SELECT * 280 | FROM VARIABLE 281 | LIMIT 5 282 | 283 | Query id: 43b29497-c340-4ee5-beed-769e56f46d62 284 | 285 | ┌─ID─────┬─NAME_─┬─VALUE_─┬──────────────TIMESTAMP_─┬─PARTITION_ID_─┬─POSITION_─┬─PROCESS_INSTANCE_KEY_─┬─PROCESS_DEFINITION_KEY_─┬───────SCOPE_KEY_─┬─STATE_──┐ 286 | │ 1-5544 │ x │ "1" │ 2023-03-12 04:14:37.390 │ 1 │ 5544 │ 2251799813687405 │ 2251799813687401 │ 2251799813687407 │ created │ 287 | │ 1-5545 │ y │ "2" │ 2023-03-12 04:14:37.390 │ 1 │ 5545 │ 2251799813687405 │ 2251799813687401 │ 2251799813687407 │ created │ 288 | │ 1-5548 │ y │ "2" │ 2023-03-12 04:14:37.390 │ 1 │ 5548 │ 2251799813687405 │ 2251799813687401 │ 2251799813687405 │ created │ 289 | │ 1-5549 │ x │ "1" │ 2023-03-12 04:14:37.390 │ 1 │ 5549 │ 2251799813687405 │ 2251799813687401 │ 2251799813687405 │ created │ 290 | │ 1-5566 │ x │ "1" │ 2023-03-12 04:15:39.281 │ 1 │ 5566 │ 2251799813687416 │ 2251799813687401 │ 2251799813687418 │ created │ 291 | └────────┴───────┴────────┴─────────────────────────┴───────────────┴───────────┴───────────────────────┴─────────────────────────┴──────────────────┴─────────┘ 292 | 293 | 5 rows in set. Elapsed: 0.004 sec. 294 | 295 | chk :) 296 | ``` 297 | 298 | ## ClickHouse Query Signal Subscription Records 299 | ``` 300 | chk :) select * from SIGNAL_SUBSCRIPTION order by PROCESS_DEFINITION_KEY_ asc,TIMESTAMP_ asc; 301 | 302 | SELECT * 303 | FROM SIGNAL_SUBSCRIPTION 304 | ORDER BY 305 | PROCESS_DEFINITION_KEY_ ASC, 306 | TIMESTAMP_ ASC 307 | 308 | Query id: bda77cac-a74a-4123-9f52-f6c14e682c5e 309 | 310 | ┌─ID_──────────────────────────────────┬─SIGNAL_NAME_───┬──────────────TIMESTAMP_─┬─STATE_──┬─CATCH_ELEMENT_INSTANCE_KEY_─┬─PROCESS_DEFINITION_KEY_─┬─BPMN_PROCESS_ID_─┬─CATCH_EVENT_ID_─┐ 311 | │ 7f43d9eb-6fb9-49f0-90b2-4920560413d4 │ Signal_1mcst4h │ 2023-03-12 14:52:33.197 │ deleted │ -1 │ 2251799813692399 │ Process_1udqk7z │ StartEvent_1 │ 312 | │ 2307850b-9505-49aa-a368-4f9431d36c5f │ Signal_1mcst4h │ 2023-03-12 14:52:33.197 │ created │ -1 │ 2251799813692405 │ Process_1udqk7z │ StartEvent_1 │ 313 | │ 7d723b2e-56bd-4d1b-9b07-dafe0ad95e11 │ Signal_3h5lmif │ 2023-03-12 14:54:03.718 │ created │ -1 │ 2251799813692408 │ Process_0k1nnh5 │ StartEvent_1 │ 314 | │ c8603d0a-7935-4bb4-8570-970aa2d0c5a2 │ Signal_3h5lmif │ 2023-03-12 14:54:21.340 │ deleted │ -1 │ 2251799813692408 │ Process_0k1nnh5 │ StartEvent_1 │ 315 | │ eee80948-95d6-402a-9ea6-4657d5aaf1f2 │ Signal_3h5lmif │ 2023-03-12 14:54:21.340 │ created │ -1 │ 2251799813692411 │ Process_0k1nnh5 │ StartEvent_1 │ 316 | └──────────────────────────────────────┴────────────────┴─────────────────────────┴─────────┴─────────────────────────────┴─────────────────────────┴──────────────────┴─────────────────┘ 317 | 318 | 5 rows in set. Elapsed: 0.003 sec. 319 | 320 | chk :) 321 | ``` 322 | -------------------------------------------------------------------------------- /docker/application.yaml: -------------------------------------------------------------------------------- 1 | zeebe: 2 | broker: 3 | exporters: 4 | clickhouse: 5 | className: io.zeebe.clickhouse.exporter.ClickHouseExporter 6 | jarPath: exporters/zeebe-clickhouse-exporter-1.0-SNAPSHOT.jar 7 | 8 | -------------------------------------------------------------------------------- /docker/clickhouse-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 25 | trace 26 | /var/log/clickhouse-server/clickhouse-server.log 27 | /var/log/clickhouse-server/clickhouse-server.err.log 28 | 31 | 1000M 32 | 10 33 | 34 | 35 | 40 | 45 | 46 | 51 | 63 | 70 | 71 | 72 | 73 | 74 | 75 | 93 | 94 | 97 | 98 | 99 | 103 | 8123 104 | 105 | 112 | 9000 113 | 114 | 117 | 9004 118 | 119 | 122 | 9005 123 | 124 | 128 | 129 | 130 | 134 | 135 | 136 | 140 | 141 | 142 | 149 | 9009 150 | 151 | 156 | 157 | 158 | 163 | 164 | 167 | 168 | 172 | 176 | 177 | 187 | :: 188 | 189 | 190 | 191 | 192 | 193 | 194 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 208 | 209 | 210 | 211 | 212 | 4096 213 | 214 | 215 | 3 216 | 217 | 218 | 219 | 220 | false 221 | 222 | 223 | /path/to/ssl_cert_file 224 | /path/to/ssl_key_file 225 | 226 | 227 | false 228 | 229 | 230 | /path/to/ssl_ca_cert_file 231 | 232 | 234 | none 235 | 236 | 237 | 0 238 | 239 | 240 | -1 241 | -1 242 | 243 | 244 | false 245 | 246 | 247 | 248 | 249 | 250 | 251 | 253 | 258 | 259 | none 260 | true 261 | true 262 | sslv2,sslv3 263 | true 264 | 265 | 266 | 267 | true 268 | true 269 | sslv2,sslv3 270 | true 271 | 272 | 273 | 274 | RejectCertificateHandler 275 | 276 | 277 | 278 | 279 | 280 | 283 | 284 | 289 | 0 290 | 291 | 292 | 100 293 | 294 | 305 | 0 306 | 307 | 313 | 314 | 10000 315 | 316 | 318 | 319 | 320 | 322 | 0.9 323 | 324 | 328 | 4194304 329 | 330 | 336 | 0 337 | 338 | 340 | 341 | 342 | 351 | 8589934592 352 | 353 | 357 | 5368709120 358 | 359 | 360 | 376 | 1000 377 | 378 | 379 | 134217728 380 | 381 | 382 | 10000 383 | 384 | 385 | /var/lib/clickhouse/ 386 | 387 | 388 | 438 | 439 | 440 | 441 | /var/lib/clickhouse/tmp/ 442 | 443 | 444 | 445 | ` 446 | 447 | 456 | 457 | 458 | 459 | /var/lib/clickhouse/user_files/ 460 | 461 | 462 | 463 | 528 | 529 | 530 | 550 | 551 | 552 | 553 | 554 | 555 | users.xml 556 | 557 | 558 | 559 | /var/lib/clickhouse/access/ 560 | 561 | 562 | 618 | 619 | 620 | 621 | 625 | false 626 | 627 | 629 | false 630 | 631 | 637 | false 638 | 639 | 642 | false 643 | 644 | 645 | 646 | default 647 | 648 | 649 | 650 | 651 | 652 | 653 | 654 | 658 | 659 | 660 | 661 | default 662 | 663 | 675 | Asia/Shanghai 676 | 677 | 680 | 681 | 682 | 686 | true 687 | 688 | 689 | false 690 | 691 | ' | sed -e 's|.*>\(.*\)<.*|\1|') 698 | wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb 699 | apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb 700 | clickhouse-jdbc-bridge & 701 | 702 | * [CentOS/RHEL] 703 | export MVN_URL=https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc-bridge/ 704 | export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') 705 | wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm 706 | yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm 707 | clickhouse-jdbc-bridge & 708 | 709 | Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. 710 | ]]> 711 | 717 | 718 | 721 | 722 | 723 | 724 | 744 | 745 | 746 | 747 | 748 | 749 | 750 | 751 | 752 | localhost 753 | 9000 754 | 755 | 756 | 757 | 758 | 759 | 760 | 761 | false 762 | 763 | 127.0.0.1 764 | 9000 765 | 766 | 767 | 127.0.0.2 768 | 9000 769 | 770 | 771 | 127.0.0.3 772 | 9000 773 | 774 | 775 | 790 | 791 | 792 | 793 | 794 | localhost 795 | 9000 796 | 797 | 798 | 799 | 800 | localhost 801 | 9000 802 | 803 | 804 | 805 | 806 | 807 | 808 | 127.0.0.1 809 | 9000 810 | 811 | 812 | 813 | 814 | 127.0.0.2 815 | 9000 816 | 817 | 818 | 819 | 820 | 821 | true 822 | 823 | 127.0.0.1 824 | 9000 825 | 826 | 827 | 828 | true 829 | 830 | 127.0.0.2 831 | 9000 832 | 833 | 834 | 835 | 836 | 837 | 838 | localhost 839 | 9440 840 | 1 841 | 842 | 843 | 844 | 845 | 846 | 847 | localhost 848 | 9000 849 | 850 | 851 | 852 | 853 | localhost 854 | 1 855 | 856 | 857 | 858 | 859 | 860 | 863 | 864 | 874 | 875 | 879 | 880 | 881 | 885 | 886 | 891 | 892 | 908 | 909 | 914 | 920 | 921 | 922 | 923 | 3600 924 | 925 | 926 | 927 | 3600 928 | 929 | 930 | 60 931 | 932 | 933 | 941 | 968 | 969 | 970 | 978 | 989 | 990 | 991 | 992 | 996 | system 997 | query_log
998 | 1006 | toYYYYMM(event_date) 1007 | 1016 | 1017 | 1020 | 1021 | 1022 | 7500 1023 | 1024 | 1025 | 1027 | 1028 | system 1029 | trace_log
1030 | 1031 | toYYYYMM(event_date) 1032 | 7500 1033 | 1034 | 1035 | 1037 | 1038 | system 1039 | query_thread_log
1040 | toYYYYMM(event_date) 1041 | 7500 1042 | 1043 | 1044 | 1046 | 1047 | system 1048 | query_views_log
1049 | toYYYYMM(event_date) 1050 | 7500 1051 | 1052 | 1053 | 1055 | 1056 | system 1057 | part_log
1058 | toYYYYMM(event_date) 1059 | 7500 1060 | 1061 | 1062 | 1072 | 1073 | 1074 | 1075 | system 1076 | metric_log
1077 | 7500 1078 | 1000 1079 | 1080 | 1081 | 1085 | 1086 | system 1087 | asynchronous_metric_log
1088 | 1092 | 7000 1093 | 1094 | 1095 | 1098 | 1099 | 1109 | 1110 | engine MergeTree 1111 | partition by toYYYYMM(finish_date) 1112 | order by (finish_date, finish_time_us, trace_id) 1113 | 1114 | system 1115 | opentelemetry_span_log
1116 | 7500 1117 | 1118 | 1119 | 1120 | 1122 | 1123 | system 1124 | crash_log
1125 | 1126 | 1127 | 1000 1128 | 1129 | 1130 | 1134 | 1141 | 1142 | 1143 | 1144 | system 1145 | processors_profile_log
1146 | 1147 | toYYYYMM(event_date) 1148 | 7500 1149 | 1150 | 1151 | 1152 | 1158 | 1159 | 1162 | 1163 | 1164 | 1167 | *_dictionary.xml 1168 | 1169 | 1170 | *_function.xml 1171 | 1172 | 1175 | 1189 | 1190 | 1195 | 1196 | 1197 | 1198 | 1199 | 1200 | 1201 | 1202 | 1203 | 1204 | 1205 | 1206 | 1207 | 1208 | 1209 | 1210 | 1211 | 1212 | 1213 | 1214 | 1215 | 1216 | 1217 | 1218 | 1220 | 1221 | 1222 | /clickhouse/task_queue/ddl 1223 | 1224 | 1225 | 1226 | 1227 | 1228 | 1229 | 1230 | 1233 | 1234 | 1235 | 1236 | 1237 | 1238 | 1239 | 1240 | 1241 | 1242 | 1243 | 1244 | 1245 | 1250 | 1251 | 1258 | 1259 | 1260 | 1261 | 1262 | 1263 | 1264 | click_cost 1265 | any 1266 | 1267 | 0 1268 | 3600 1269 | 1270 | 1271 | 86400 1272 | 60 1273 | 1274 | 1275 | 1276 | max 1277 | 1278 | 0 1279 | 60 1280 | 1281 | 1282 | 3600 1283 | 300 1284 | 1285 | 1286 | 86400 1287 | 3600 1288 | 1289 | 1290 | 1291 | 1292 | 1295 | /var/lib/clickhouse/format_schemas/ 1296 | 1297 | 1303 | 1304 | 1305 | hide encrypt/decrypt arguments 1306 | ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) 1307 | 1310 | \1(???) 1311 | 1312 | 1313 | 1314 | 1357 | 1358 | 1359 | 1360 | 1361 | 1362 | false 1363 | 1364 | false 1365 | 1366 | 1367 | https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 1368 | 1369 | 1370 | 1371 | 1372 | 1373 | 1374 | 1395 | 1396 | 1397 | 1401 | 1402 | 1410 | 1411 | -------------------------------------------------------------------------------- /docker/clickhouse-users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 17 | random 18 | 19 | 20 | 21 | 22 | 1 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 65 | clickhouse123 66 | 67 | 87 | 88 | ::/0 89 | 90 | 91 | 92 | default 93 | 94 | 95 | default 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 3600 110 | 111 | 112 | 0 113 | 0 114 | 0 115 | 0 116 | 0 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | networks: 4 | zeebe_network: 5 | driver: bridge 6 | 7 | services: 8 | clickhouse: 9 | container_name: clickhouse 10 | image: clickhouse/clickhouse-server:24.11.1.2557 11 | ulimits: 12 | nofile: 13 | soft: 262144 14 | hard: 262144 15 | ports: 16 | - "8123:8123" 17 | - "19600:9600" 18 | volumes: 19 | - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml 20 | - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml 21 | healthcheck: 22 | test: [ "CMD-SHELL", "curl -f http://localhost:8123/ | grep -q Ok" ] 23 | interval: 30s 24 | timeout: 5s 25 | retries: 3 26 | start_period: 30s 27 | networks: 28 | - zeebe_network 29 | 30 | zeebe: 31 | container_name: zeebe_broker 32 | image: camunda/zeebe:8.6.5 33 | environment: 34 | - ZEEBE_LOG_LEVEL=debug 35 | - ZEEBE_CLICKHOUSE_URL=jdbc:ch://clickhouse/default 36 | - ZEEBE_CLICKHOUSE_USER=default 37 | - ZEEBE_CLICKHOUSE_PASSWORD=clickhouse123 38 | ports: 39 | - "26500:26500" 40 | - "9600:9600" 41 | volumes: 42 | - ../target/zeebe-clickhouse-exporter-1.0-SNAPSHOT.jar:/usr/local/zeebe/exporters/zeebe-clickhouse-exporter-1.0-SNAPSHOT.jar 43 | - ./application.yaml:/usr/local/zeebe/config/application.yaml 44 | networks: 45 | - zeebe_network 46 | depends_on: 47 | - clickhouse 48 | -------------------------------------------------------------------------------- /how-it-works.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camunda-community-hub/zeebe-clickhouse-exporter/5f01980bb6f243b2aee43ccc231266c8b6d82346/how-it-works.jpg -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | io.zeebe.clickhouse 5 | zeebe-clickhouse-exporter 6 | jar 7 | 1.0-SNAPSHOT 8 | zeebe-clickhouse-exporter 9 | 10 | 11 | org.camunda.community 12 | community-hub-release-parent 13 | 1.4.4 14 | 15 | 16 | 17 | 18 | 19 | Zeebe Community License v1.1 20 | https://zeebe.io/legal/zeebe-community-license-1.1 21 | 22 | 23 | 24 | 25 | 26 | UTF-8 27 | 17 28 | 29 | https://artifacts.camunda.com/artifactory/zeebe-io-snapshots/ 30 | 31 | https://artifacts.camunda.com/artifactory/zeebe-io/ 32 | 33 | 34 | 35 | 36 | ${maven.multiModuleProjectDirectory}/COPYING-HEADER.txt 37 | 38 | 39 | 8.6.5 40 | 2.20.0 41 | 5.11.3 42 | 3.26.3 43 | 1.20.4 44 | 3.6.5 45 | 4.2.2 46 | 10.20.2 47 | 0.28.1 48 | 0.6.5 49 | 2.17.0 50 | 3.17.0 51 | 52 | 53 | 1.24 54 | 3.7.1 55 | 3.13.0 56 | 3.5.0 57 | 3.4.0 58 | 3.1.2 59 | 3.5.0 60 | 3.5.2 61 | 2.24 62 | 3.2.7 63 | 3.11.1 64 | 4.6 65 | 0.6.1 66 | 1.0.6 67 | 0.15.0 68 | 3.5.2 69 | 70 | 71 | 1.6.1 72 | 73 | 74 | -Xdoclint:none 75 | 76 | 82 | false 83 | ${skipChecks} 84 | ${skipChecks} 85 | ${skipChecks} 86 | ${skipChecks} 87 | ${skipChecks} 88 | ${skipChecks} 89 | 90 | 91 | 92 | 93 | 94 | io.camunda 95 | zeebe-bom 96 | ${version.zeebe} 97 | import 98 | pom 99 | 100 | 101 | org.junit 102 | junit-bom 103 | ${version.junit} 104 | pom 105 | import 106 | 107 | 108 | org.testcontainers 109 | testcontainers-bom 110 | ${version.testcontainers} 111 | pom 112 | import 113 | 114 | 115 | 116 | 117 | 118 | io.camunda 119 | zeebe-exporter-api 120 | provided 121 | 122 | 123 | 124 | com.clickhouse 125 | clickhouse-jdbc 126 | ${version.clickhouse-jdbc} 127 | http 128 | 129 | 130 | * 131 | * 132 | 133 | 134 | 135 | 136 | commons-io 137 | commons-io 138 | ${version.commons-io} 139 | 140 | 141 | org.apache.commons 142 | commons-lang3 143 | ${version.commons-lang3} 144 | 145 | 146 | 147 | 148 | org.testcontainers 149 | testcontainers 150 | test 151 | 152 | 153 | org.testcontainers 154 | clickhouse 155 | test 156 | 157 | 158 | io.zeebe 159 | zeebe-test-container 160 | ${version.zeebe-test-container} 161 | test 162 | 163 | 164 | org.testcontainers 165 | junit-jupiter 166 | test 167 | 168 | 169 | org.junit.jupiter 170 | junit-jupiter-api 171 | test 172 | 173 | 174 | org.assertj 175 | assertj-core 176 | ${versin.assertj} 177 | test 178 | 179 | 180 | org.awaitility 181 | awaitility 182 | ${version.awaitility} 183 | test 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | org.apache.maven.plugins 193 | maven-compiler-plugin 194 | ${plugin.version.compiler} 195 | 196 | 197 | 198 | 199 | org.apache.maven.plugins 200 | maven-javadoc-plugin 201 | ${plugin.version.javadoc} 202 | 203 | ${version.java} 204 | 205 | 206 | 207 | 208 | 209 | com.mycila 210 | license-maven-plugin 211 | ${plugin.version.license} 212 | 213 |
${license.header}
214 | 215 | camunda services GmbH 216 | info@camunda.com 217 | 218 | 219 | **/*.java 220 | 221 | 222 | SLASHSTAR_STYLE 223 | 224 | 225 | 226 | 227 | add-license 228 | compile 229 | 230 | format 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | org.apache.maven.plugins 239 | maven-checkstyle-plugin 240 | ${plugin.version.checkstyle} 241 | 242 | 243 | 244 | io.camunda 245 | zeebe-build-tools 246 | ${version.zeebe} 247 | 248 | 249 | com.puppycrawl.tools 250 | checkstyle 251 | ${version.checkstyle} 252 | 253 | 254 | 255 | check/.checkstyle.xml 256 | true 257 | UTF-8 258 | true 259 | 260 | ${project.build.sourceDirectory} 261 | ${project.build.testSourceDirectory} 262 | 263 | 264 | 265 | 266 | validate-java 267 | validate 268 | 269 | check 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | org.apache.maven.plugins 280 | maven-surefire-plugin 281 | ${plugin.version.surefire} 282 | 283 | false 284 | false 285 | true 286 | 287 | 288 | junit.jupiter.execution.parallel.enabled = true 289 | junit.jupiter.execution.parallel.mode.default = same_thread 290 | junit.jupiter.execution.parallel.config.strategy = dynamic 291 | junit.jupiter.execution.parallel.config.dynamic.factor = 2 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | org.apache.maven.plugins 300 | maven-failsafe-plugin 301 | ${plugin.version.failsafe} 302 | 303 | false 304 | false 305 | true 306 | 307 | 308 | junit.jupiter.execution.parallel.enabled = true 309 | junit.jupiter.execution.parallel.mode.default = same_thread 310 | junit.jupiter.execution.parallel.config.strategy = dynamic 311 | junit.jupiter.execution.parallel.config.dynamic.factor = 2 312 | 313 | 314 | 315 | 316 | 317 | 318 | integration-test 319 | verify 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | com.spotify.fmt 328 | fmt-maven-plugin 329 | ${plugin.version.fmt} 330 | 331 | 332 | 333 | format 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | org.apache.maven.plugins 342 | maven-assembly-plugin 343 | ${plugin.version.assembly} 344 | 345 | 346 | 347 | 348 | org.apache.maven.plugins 349 | maven-enforcer-plugin 350 | ${plugin.version.enforcer} 351 | 352 | 353 | enforce-dependency-convergence 354 | 355 | enforce 356 | 357 | 358 | 359 | 360 | 361 | 362 | [11,) 363 | 364 | 365 | [3.6,) 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | org.apache.maven.plugins 376 | maven-clean-plugin 377 | ${plugin.version.clean} 378 | 379 | 380 | 381 | 382 | org.revapi 383 | revapi-maven-plugin 384 | ${plugin.version.revapi} 385 | 386 | 387 | org.revapi 388 | revapi-java 389 | ${version.revapi} 390 | 391 | 392 | 393 | 394 | verify 395 | check 396 | 397 | check 398 | 399 | 400 | 401 | 402 | true 403 | true 404 | true 405 | 406 | 407 | 408 | ${maven.multiModuleProjectDirectory}/revapi.json 409 | 410 | 411 | 412 | 413 | 414 | 415 | org.codehaus.mojo 416 | animal-sniffer-maven-plugin 417 | ${plugin.version.animal-sniffer} 418 | 419 | 420 | 421 | 422 | 423 | 424 | org.apache.maven.plugins 425 | maven-assembly-plugin 426 | 427 | 428 | jar-with-dependencies 429 | 430 | ${project.build.directory} 431 | 432 | 433 | 434 | assemble-all 435 | package 436 | 437 | single 438 | 439 | 440 | 441 | assemble-for-jib 442 | package 443 | 444 | single 445 | 446 | 447 | ${project.build.directory}/jib 448 | ${project.artifactId} 449 | 450 | 451 | 452 | 453 | 455 | 456 | org.apache.maven.plugins 457 | maven-dependency-plugin 458 | 459 | 460 | copy 461 | pre-integration-test 462 | 463 | copy 464 | 465 | 466 | 467 | 468 | io.zeebe.clickhouse 469 | zeebe-clickhouse-exporter 470 | ${project.version} 471 | jar 472 | jar-with-dependencies 473 | ${project.basedir}/src/test/resources 474 | zeebe-clickhouse-exporter.jar 475 | 476 | 477 | false 478 | true 479 | 480 | 481 | 482 | 483 | 484 | 485 | org.apache.maven.plugins 486 | maven-compiler-plugin 487 | 488 | 489 | 490 | com.mycila 491 | license-maven-plugin 492 | 493 | 494 | 495 | org.apache.maven.plugins 496 | maven-checkstyle-plugin 497 | 498 | 499 | 500 | org.apache.maven.plugins 501 | maven-surefire-plugin 502 | 503 | 504 | 505 | com.spotify.fmt 506 | fmt-maven-plugin 507 | 508 | 509 | 510 | org.apache.maven.plugins 511 | maven-enforcer-plugin 512 | 513 | 514 | 515 | org.apache.maven.plugins 516 | maven-clean-plugin 517 | 518 | 519 | 520 | 521 | 522 | community-action-maven-release 523 | 524 | true 525 | true 526 | 527 | 528 | 529 | 530 | 531 | org.apache.maven.plugins 532 | maven-gpg-plugin 533 | ${plugin.version.gpg} 534 | 535 | 536 | sign-artifacts 537 | verify 538 | 539 | sign 540 | 541 | 542 | 543 | 544 | 545 | 546 | --pinentry-mode 547 | loopback 548 | 549 | 550 | 551 | 552 | 553 | 554 | 555 | 556 | 557 | 558 | zeebe 559 | Zeebe Repository 560 | https://artifacts.camunda.com/artifactory/zeebe-io/ 561 | 562 | true 563 | 564 | 565 | false 566 | 567 | 568 | 569 | 570 | zeebe-snapshots 571 | Zeebe Snapshot Repository 572 | https://artifacts.camunda.com/artifactory/zeebe-io-snapshots/ 573 | 574 | false 575 | 576 | 577 | true 578 | 579 | 580 | 581 | 582 | 583 | https://github.com/camunda-community-hub/zeebe-clickhouse-exporter 584 | scm:git:git@github.com:camunda-community-hub/zeebe-clickhouse-exporter.git 585 | scm:git:git@github.com:camunda-community-hub/zeebe-clickhouse-exporter.git 586 | 587 | HEAD 588 | 589 | 590 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:recommended" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/ClickHouseExporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter; 9 | 10 | import io.camunda.zeebe.exporter.api.Exporter; 11 | import io.camunda.zeebe.exporter.api.context.Context; 12 | import io.camunda.zeebe.exporter.api.context.Controller; 13 | import io.camunda.zeebe.protocol.record.Record; 14 | import org.slf4j.Logger; 15 | 16 | public class ClickHouseExporter implements Exporter { 17 | 18 | private Logger logger; 19 | private Controller controller; 20 | 21 | private ExporterConfiguration configuration; 22 | private ClickHouseExporterClient client; 23 | 24 | @Override 25 | public void configure(final Context context) { 26 | logger = context.getLogger(); 27 | configuration = context.getConfiguration().instantiate(ExporterConfiguration.class); 28 | logger.info("Exporter configured with {}", configuration); 29 | } 30 | 31 | @Override 32 | public void open(final Controller controller) { 33 | this.controller = controller; 34 | client = createClient(); 35 | logger.info("Exporter opened"); 36 | } 37 | 38 | @Override 39 | public void close() {} 40 | 41 | @Override 42 | public void export(final Record record) { 43 | final long lastPosition = record.getPosition(); 44 | client.insert(record, lastPosition); 45 | controller.updateLastExportedRecordPosition(lastPosition); 46 | } 47 | 48 | protected ClickHouseExporterClient createClient() { 49 | return new ClickHouseExporterClient(configuration, logger); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/ClickHouseExporterClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter; 9 | 10 | import static io.zeebe.clickhouse.exporter.importer.ClickHouseConfig.*; 11 | 12 | import io.camunda.zeebe.protocol.record.Record; 13 | import io.camunda.zeebe.protocol.record.RecordType; 14 | import io.camunda.zeebe.protocol.record.ValueType; 15 | import io.zeebe.clickhouse.exporter.importer.*; 16 | import java.sql.SQLException; 17 | import org.slf4j.Logger; 18 | 19 | public class ClickHouseExporterClient { 20 | 21 | private final String elementInstanceTable = "ELEMENT_INSTANCE"; 22 | private final String clickHouseConfigTable = "CLICKHOUSE_CONFIG"; 23 | 24 | private final ExporterConfiguration configuration; 25 | 26 | private final Logger logger; 27 | private long cfgPosition = -1; 28 | 29 | ClickHouseExporterClient(final ExporterConfiguration configuration, final Logger logger) { 30 | this.configuration = configuration; 31 | this.logger = logger; 32 | logger.info("configuration--------------->" + configuration.toString()); 33 | try { 34 | // 初始化配置表 35 | createClickHouseConfigTable( 36 | configuration.getChUrl(), 37 | configuration.getChUser(), 38 | configuration.getChPassword(), 39 | clickHouseConfigTable); 40 | // 检查是否有初始化配置信息 41 | final long i = 42 | queryClickHouseConfig( 43 | configuration.getChUrl(), 44 | configuration.getChUser(), 45 | configuration.getChPassword(), 46 | clickHouseConfigTable); 47 | 48 | if (i <= 0L) { 49 | // 执行初始化 50 | initClickHouseConfigTable( 51 | configuration.getChUrl(), 52 | configuration.getChUser(), 53 | configuration.getChPassword(), 54 | clickHouseConfigTable); 55 | } else { 56 | cfgPosition = i; 57 | } 58 | // 创建流程定义信息表 59 | createProcessTable( 60 | configuration.getChUrl(), 61 | configuration.getChUser(), 62 | configuration.getChPassword(), 63 | ValueType.PROCESS.name()); 64 | // 创建流程实例表 65 | createProcessInstanceTable( 66 | configuration.getChUrl(), 67 | configuration.getChUser(), 68 | configuration.getChPassword(), 69 | ValueType.PROCESS_INSTANCE.name()); 70 | // 创建任务实例表 71 | createElementInstanceTable( 72 | configuration.getChUrl(), 73 | configuration.getChUser(), 74 | configuration.getChPassword(), 75 | elementInstanceTable); 76 | // 创建调度表 77 | createJobTable( 78 | configuration.getChUrl(), 79 | configuration.getChUser(), 80 | configuration.getChPassword(), 81 | ValueType.JOB.name()); 82 | // 创建流程变量表 83 | createVariableTable( 84 | configuration.getChUrl(), 85 | configuration.getChUser(), 86 | configuration.getChPassword(), 87 | ValueType.VARIABLE.name()); 88 | // 创建事件表 89 | createIncidentTable( 90 | configuration.getChUrl(), 91 | configuration.getChUser(), 92 | configuration.getChPassword(), 93 | ValueType.INCIDENT.name()); 94 | // 创建定时器表 95 | createTimerTable( 96 | configuration.getChUrl(), 97 | configuration.getChUser(), 98 | configuration.getChPassword(), 99 | ValueType.TIMER.name()); 100 | // 创建异常表 101 | createErrorTable( 102 | configuration.getChUrl(), 103 | configuration.getChUser(), 104 | configuration.getChPassword(), 105 | ValueType.ERROR.name()); 106 | // 创建消息表 107 | createMessageTable( 108 | configuration.getChUrl(), 109 | configuration.getChUser(), 110 | configuration.getChPassword(), 111 | ValueType.MESSAGE.name()); 112 | // 创建消息订阅表 113 | createMessageSubscriptionTable( 114 | configuration.getChUrl(), 115 | configuration.getChUser(), 116 | configuration.getChPassword(), 117 | ValueType.MESSAGE_SUBSCRIPTION.name()); 118 | // 创建信号订阅表 119 | createSignalSubscriptionTable( 120 | configuration.getChUrl(), 121 | configuration.getChUser(), 122 | configuration.getChPassword(), 123 | ValueType.SIGNAL_SUBSCRIPTION.name()); 124 | 125 | } catch (final SQLException e) { 126 | e.printStackTrace(); 127 | } 128 | } 129 | 130 | /** 执行数据导出 * */ 131 | public void insert(final Record record, final long lastPosition) { 132 | if ((lastPosition > cfgPosition || cfgPosition == -1L) 133 | && RecordType.EVENT.name().equals(record.getRecordType().name())) { 134 | 135 | logger.info( 136 | String.format("------%s---------->%s", record.getValueType().name(), record.toJson())); 137 | // 流程定义信息 138 | if (ValueType.PROCESS.name().equals(record.getValueType().name())) { 139 | try { 140 | 141 | ProcessImporter.batchProcessInsert( 142 | configuration.getChUrl(), 143 | configuration.getChUser(), 144 | configuration.getChPassword(), 145 | ValueType.PROCESS.name(), 146 | record); 147 | // 更新记录位置 148 | update(lastPosition); 149 | } catch (final SQLException e) { 150 | e.printStackTrace(); 151 | } 152 | } 153 | // 流程实例信息 & 任务实例信息 154 | if (ValueType.PROCESS_INSTANCE.name().equals(record.getValueType().name())) { 155 | try { 156 | 157 | // 流程实例信息 158 | ProcessInstanceImporter.batchProcessInstanceInsertOrUpdate( 159 | configuration.getChUrl(), 160 | configuration.getChUser(), 161 | configuration.getChPassword(), 162 | ValueType.PROCESS_INSTANCE.name(), 163 | record); 164 | 165 | // 任务实例信息 166 | ProcessInstanceImporter.batchElementInstanceInsert( 167 | configuration.getChUrl(), 168 | configuration.getChUser(), 169 | configuration.getChPassword(), 170 | elementInstanceTable, 171 | record); 172 | // 更新记录位置 173 | update(lastPosition); 174 | } catch (final SQLException e) { 175 | e.printStackTrace(); 176 | } 177 | } 178 | 179 | // 调度信息 180 | if (ValueType.JOB.name().equals(record.getValueType().name())) { 181 | try { 182 | 183 | JobImporter.batchJobInsertOrUpdate( 184 | configuration.getChUrl(), 185 | configuration.getChUser(), 186 | configuration.getChPassword(), 187 | ValueType.JOB.name(), 188 | record); 189 | // 更新记录位置 190 | update(lastPosition); 191 | } catch (final SQLException e) { 192 | e.printStackTrace(); 193 | } 194 | } 195 | // 流程变量信息 196 | if (ValueType.VARIABLE.name().equals(record.getValueType().name())) { 197 | try { 198 | 199 | VariableImporter.batchVariableInsert( 200 | configuration.getChUrl(), 201 | configuration.getChUser(), 202 | configuration.getChPassword(), 203 | ValueType.VARIABLE.name(), 204 | record); 205 | // 更新记录位置 206 | update(lastPosition); 207 | } catch (final SQLException e) { 208 | e.printStackTrace(); 209 | } 210 | } 211 | // 事件信息 212 | if (ValueType.INCIDENT.name().equals(record.getValueType().name())) { 213 | try { 214 | 215 | IncidentImporter.batchIncidentInsertOrUpdate( 216 | configuration.getChUrl(), 217 | configuration.getChUser(), 218 | configuration.getChPassword(), 219 | ValueType.INCIDENT.name(), 220 | record); 221 | // 更新记录位置 222 | update(lastPosition); 223 | } catch (final SQLException e) { 224 | e.printStackTrace(); 225 | } 226 | } 227 | // 定时器信息 228 | if (ValueType.TIMER.name().equals(record.getValueType().name())) { 229 | try { 230 | 231 | TimerImporter.batchTimerInsert( 232 | configuration.getChUrl(), 233 | configuration.getChUser(), 234 | configuration.getChPassword(), 235 | ValueType.TIMER.name(), 236 | record); 237 | // 更新记录位置 238 | update(lastPosition); 239 | } catch (final SQLException e) { 240 | e.printStackTrace(); 241 | } 242 | } 243 | // 异常记录信息 244 | if (ValueType.ERROR.name().equals(record.getValueType().name())) { 245 | try { 246 | 247 | ErrorImporter.batchErrorInsert( 248 | configuration.getChUrl(), 249 | configuration.getChUser(), 250 | configuration.getChPassword(), 251 | ValueType.ERROR.name(), 252 | record); 253 | // 更新记录位置 254 | update(lastPosition); 255 | } catch (final SQLException e) { 256 | e.printStackTrace(); 257 | } 258 | } 259 | // 消息记录信息 260 | if (ValueType.MESSAGE.name().equals(record.getValueType().name())) { 261 | try { 262 | 263 | MessageImporter.batchMessageInsert( 264 | configuration.getChUrl(), 265 | configuration.getChUser(), 266 | configuration.getChPassword(), 267 | ValueType.MESSAGE.name(), 268 | record); 269 | // 更新记录位置 270 | update(lastPosition); 271 | } catch (final SQLException e) { 272 | e.printStackTrace(); 273 | } 274 | } 275 | // 消息订阅记录信息 276 | if (ValueType.MESSAGE_SUBSCRIPTION.name().equals(record.getValueType().name())) { 277 | try { 278 | 279 | MessageSubscriptionImporter.batchMessageSubscriptionInsert( 280 | configuration.getChUrl(), 281 | configuration.getChUser(), 282 | configuration.getChPassword(), 283 | ValueType.MESSAGE_SUBSCRIPTION.name(), 284 | record); 285 | // 更新记录位置 286 | update(lastPosition); 287 | } catch (final SQLException e) { 288 | e.printStackTrace(); 289 | } 290 | } 291 | // 消息启动记录信息 292 | if (ValueType.MESSAGE_START_EVENT_SUBSCRIPTION.name().equals(record.getValueType().name())) { 293 | try { 294 | 295 | MessageSubscriptionImporter.batchMessageStartEventSubscriptionInsert( 296 | configuration.getChUrl(), 297 | configuration.getChUser(), 298 | configuration.getChPassword(), 299 | ValueType.MESSAGE_SUBSCRIPTION.name(), 300 | record); 301 | // 更新记录位置 302 | update(lastPosition); 303 | } catch (final SQLException e) { 304 | e.printStackTrace(); 305 | } 306 | } 307 | // 信号启动记录信息 308 | if (ValueType.SIGNAL_SUBSCRIPTION.name().equals(record.getValueType().name())) { 309 | try { 310 | 311 | SignalSubscriptionImporter.batchSignalSubscriptionInsert( 312 | configuration.getChUrl(), 313 | configuration.getChUser(), 314 | configuration.getChPassword(), 315 | ValueType.SIGNAL_SUBSCRIPTION.name(), 316 | record); 317 | // 更新记录位置 318 | update(lastPosition); 319 | } catch (final SQLException e) { 320 | e.printStackTrace(); 321 | } 322 | } 323 | } 324 | } 325 | 326 | // 更新记录位置 327 | public void update(final long lastPosition) { 328 | try { 329 | updateClickHouseConfigTable( 330 | configuration.getChUrl(), 331 | configuration.getChUser(), 332 | configuration.getChPassword(), 333 | clickHouseConfigTable, 334 | lastPosition); 335 | } catch (final SQLException e) { 336 | e.printStackTrace(); 337 | } 338 | } 339 | } 340 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/ExporterConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter; 9 | 10 | import java.util.Optional; 11 | 12 | public class ExporterConfiguration { 13 | private static final String ENV_PREFIX = "ZEEBE_CLICKHOUSE_"; 14 | private final String chUrl = "jdbc:ch://127.0.0.1:8123/default"; 15 | private final String chUser = "default"; 16 | private final String chPassword = ""; 17 | 18 | public String getChUrl() { 19 | return getEnv("URL").orElse(chUrl); 20 | } 21 | 22 | public String getChUser() { 23 | return getEnv("USER").orElse(chUser); 24 | } 25 | 26 | public String getChPassword() { 27 | return getEnv("PASSWORD").orElse(chPassword); 28 | } 29 | 30 | private Optional getEnv(final String name) { 31 | return Optional.ofNullable(System.getenv(ENV_PREFIX + name)); 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return "ExporterConfiguration{" 37 | + "chUrl='" 38 | + chUrl 39 | + '\'' 40 | + ", chUser='" 41 | + chUser 42 | + '\'' 43 | + ", chPassword='" 44 | + chPassword 45 | + '\'' 46 | + '}'; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/ClickHouseConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import java.sql.*; 11 | 12 | public class ClickHouseConfig { 13 | 14 | public static void createClickHouseConfigTable( 15 | final String url, final String user, final String password, final String table) 16 | throws SQLException { 17 | try (final Connection conn = DriverManager.getConnection(url, user, password); 18 | final Statement stmt = conn.createStatement()) { 19 | stmt.execute( 20 | String.format( 21 | "create table if not exists %1$s (KEY_ String,POSITION_ Int64) " 22 | + " engine = MergeTree() ORDER BY KEY_", 23 | table)); 24 | } 25 | } 26 | 27 | public static void initClickHouseConfigTable( 28 | final String url, final String user, final String password, final String table) 29 | throws SQLException { 30 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 31 | final String sql = String.format("insert into %1$s values(?, ?)", table); 32 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 33 | ps.setString(1, "cfg"); 34 | ps.setLong(2, -1); 35 | ps.execute(); 36 | } 37 | } 38 | } 39 | 40 | public static void updateClickHouseConfigTable( 41 | final String url, 42 | final String user, 43 | final String password, 44 | final String table, 45 | final long lastPosition) 46 | throws SQLException { 47 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 48 | final String sql = 49 | String.format("alter table %1$s update POSITION_ = ? where KEY_ = ?", table); 50 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 51 | ps.setLong(1, lastPosition); 52 | 53 | ps.setString(2, "cfg"); 54 | ps.execute(); 55 | } 56 | } 57 | } 58 | 59 | public static long queryClickHouseConfig( 60 | final String url, final String user, final String password, final String table) 61 | throws SQLException { 62 | try (final Connection conn = DriverManager.getConnection(url, user, password); 63 | final Statement stmt = conn.createStatement(); 64 | final ResultSet rs = stmt.executeQuery("select * from " + table)) { 65 | long count = 0; 66 | long cfgposition = -1L; 67 | while (rs.next()) { 68 | count++; 69 | cfgposition = rs.getLong("POSITION_"); 70 | } 71 | if (count > 0) { 72 | count = cfgposition; 73 | } 74 | return count; 75 | } 76 | } 77 | 78 | public static void createProcessTable( 79 | final String url, final String user, final String password, final String table) 80 | throws SQLException { 81 | try (final Connection conn = DriverManager.getConnection(url, user, password); 82 | final Statement stmt = conn.createStatement()) { 83 | stmt.execute( 84 | String.format( 85 | "create table if not exists %1$s (KEY_ UInt64,BPMN_PROCESS_ID_ String, RESOURCE_NAME_ String," 86 | + " TIMESTAMP_ DateTime64(3), PARTITION_ID_ UInt16, POSITION_ UInt64,VERSION_ UInt16,RESOURCE_ String)" 87 | + " engine = ReplacingMergeTree(TIMESTAMP_) PARTITION BY toYYYYMM(TIMESTAMP_) ORDER BY KEY_", 88 | table)); 89 | } 90 | } 91 | 92 | public static void createProcessInstanceTable( 93 | final String url, final String user, final String password, final String table) 94 | throws SQLException { 95 | try (final Connection conn = DriverManager.getConnection(url, user, password); 96 | final Statement stmt = conn.createStatement()) { 97 | stmt.execute( 98 | String.format( 99 | "create table if not exists %1$s (KEY_ UInt64,BPMN_PROCESS_ID_ String," 100 | + " PROCESS_DEFINITION_KEY_ UInt64, START_ DateTime64(3), " 101 | + " END_ Nullable(DateTime64(3)),PARTITION_ID_ UInt16," 102 | + " VERSION_ UInt16,STATE_ String, PARENT_PROCESS_INSTANCE_KEY_ Int64," 103 | + " PARENT_ELEMENT_INSTANCE_KEY_ Int64)" 104 | + " engine = ReplacingMergeTree(START_) PARTITION BY toYYYYMM(START_) ORDER BY KEY_", 105 | table)); 106 | } 107 | } 108 | 109 | public static void createErrorTable( 110 | final String url, final String user, final String password, final String table) 111 | throws SQLException { 112 | try (final Connection conn = DriverManager.getConnection(url, user, password); 113 | final Statement stmt = conn.createStatement()) { 114 | stmt.execute( 115 | String.format( 116 | "create table if not exists %1$s (POSITION_ UInt64,ERROR_EVENT_POSITION_ UInt64," 117 | + " TIMESTAMP_ DateTime64(3)," 118 | + " PROCESS_INSTANCE_KEY_ UInt64,EXCEPTION_MESSAGE_ String, STACKTRACE_ String) " 119 | + " engine = ReplacingMergeTree(TIMESTAMP_) PARTITION BY toYYYYMM(TIMESTAMP_) ORDER BY POSITION_", 120 | table)); 121 | } 122 | } 123 | 124 | public static void createTimerTable( 125 | final String url, final String user, final String password, final String table) 126 | throws SQLException { 127 | try (final Connection conn = DriverManager.getConnection(url, user, password); 128 | final Statement stmt = conn.createStatement()) { 129 | stmt.execute( 130 | String.format( 131 | "create table if not exists %1$s (KEY_ UInt64,STATE_ String, REPETITIONS Int16," 132 | + " TIMESTAMP_ DateTime64(3),DUE_DATE_ DateTime64(3)," 133 | + " PROCESS_INSTANCE_KEY_ Int64,PROCESS_DEFINITION_KEY_ UInt64," 134 | + " ELEMENT_INSTANCE_KEY_ Int64, TARGET_ELEMENT_ID_ String) " 135 | + " engine = ReplacingMergeTree(TIMESTAMP_) PARTITION BY toYYYYMM(TIMESTAMP_) ORDER BY KEY_", 136 | table)); 137 | } 138 | } 139 | 140 | public static void createMessageTable( 141 | final String url, final String user, final String password, final String table) 142 | throws SQLException { 143 | try (final Connection conn = DriverManager.getConnection(url, user, password); 144 | final Statement stmt = conn.createStatement()) { 145 | stmt.execute( 146 | String.format( 147 | "create table if not exists %1$s (KEY_ UInt64,NAME_ String," 148 | + " TIMESTAMP_ DateTime64(3),STATE_ String," 149 | + " CORRELATION_KEY_ String,MESSAGE_ID_ String, PAYLOAD_ String) " 150 | + " engine = ReplacingMergeTree(TIMESTAMP_) PARTITION BY toYYYYMM(TIMESTAMP_) ORDER BY KEY_", 151 | table)); 152 | } 153 | } 154 | 155 | public static void createMessageSubscriptionTable( 156 | final String url, final String user, final String password, final String table) 157 | throws SQLException { 158 | try (final Connection conn = DriverManager.getConnection(url, user, password); 159 | final Statement stmt = conn.createStatement()) { 160 | stmt.execute( 161 | String.format( 162 | "create table if not exists %1$s (ID_ String,MESSAGE_NAME_ String,MESSAGE_KEY_ Int64," 163 | + " TIMESTAMP_ DateTime64(3),STATE_ String,PROCESS_INSTANCE_KEY_ Int64," 164 | + " ELEMENT_INSTANCE_KEY_ Int64,PROCESS_DEFINITION_KEY_ Int64," 165 | + " CORRELATION_KEY_ Nullable(String),TARGET_FLOW_NODE_ID_ Nullable(String)) " 166 | + " engine = ReplacingMergeTree(TIMESTAMP_) PARTITION BY toYYYYMM(TIMESTAMP_) ORDER BY ID_", 167 | table)); 168 | } 169 | } 170 | 171 | public static void createElementInstanceTable( 172 | final String url, final String user, final String password, final String table) 173 | throws SQLException { 174 | try (final Connection conn = DriverManager.getConnection(url, user, password); 175 | final Statement stmt = conn.createStatement()) { 176 | stmt.execute( 177 | String.format( 178 | "create table if not exists %1$s (ID String,KEY_ UInt64,BPMN_PROCESS_ID_ String," 179 | + " PROCESS_DEFINITION_KEY_ UInt64, TIMESTAMP_ DateTime64(3), " 180 | + " INTENT_ String,PARTITION_ID_ UInt16," 181 | + " POSITION_ UInt64, PROCESS_INSTANCE_KEY_ UInt64," 182 | + " FLOW_SCOPE_KEY_ Int64,ELEMENT_ID_ String,BPMN_ELEMENT_TYPE_ String)" 183 | + " engine = ReplacingMergeTree(TIMESTAMP_) PARTITION BY toYYYYMM(TIMESTAMP_) ORDER BY (ID,INTENT_)", 184 | table)); 185 | } 186 | } 187 | 188 | public static void createJobTable( 189 | final String url, final String user, final String password, final String table) 190 | throws SQLException { 191 | try (final Connection conn = DriverManager.getConnection(url, user, password); 192 | final Statement stmt = conn.createStatement()) { 193 | stmt.execute( 194 | String.format( 195 | "create table if not exists %1$s (KEY_ UInt64,BPMN_PROCESS_ID_ String, ELEMENT_ID_ String," 196 | + " WORKER_ String, JOB_TYPE_ String, STATE_ String, RETRIES_ UInt8, " 197 | + " START_ DateTime64(3),END_ Nullable(DateTime64(3)), PROCESS_INSTANCE_KEY_ UInt64," 198 | + " ELEMENT_INSTANCE_KEY_ UInt64, PROCESS_DEFINITION_KEY_ UInt64)" 199 | + " engine = ReplacingMergeTree(START_) PARTITION BY toYYYYMM(START_) ORDER BY KEY_", 200 | table)); 201 | } 202 | } 203 | 204 | public static void createVariableTable( 205 | final String url, final String user, final String password, final String table) 206 | throws SQLException { 207 | try (final Connection conn = DriverManager.getConnection(url, user, password); 208 | final Statement stmt = conn.createStatement()) { 209 | stmt.execute( 210 | String.format( 211 | "create table if not exists %1$s (ID String,NAME_ String, VALUE_ String," 212 | + " TIMESTAMP_ DateTime64(3), PARTITION_ID_ UInt16, POSITION_ UInt64," 213 | + " PROCESS_INSTANCE_KEY_ UInt64,PROCESS_DEFINITION_KEY_ UInt64," 214 | + " SCOPE_KEY_ Int64, STATE_ String) " 215 | + " engine = ReplacingMergeTree(TIMESTAMP_) PARTITION BY toYYYYMM(TIMESTAMP_) ORDER BY (PARTITION_ID_,POSITION_)", 216 | table)); 217 | } 218 | } 219 | 220 | public static void createIncidentTable( 221 | final String url, final String user, final String password, final String table) 222 | throws SQLException { 223 | try (final Connection conn = DriverManager.getConnection(url, user, password); 224 | final Statement stmt = conn.createStatement()) { 225 | stmt.execute( 226 | String.format( 227 | "create table if not exists %1$s (KEY_ UInt64,BPMN_PROCESS_ID_ String, ERROR_MSG_ String," 228 | + " ERROR_TYPE_ String, " 229 | + " CREATED_ DateTime64(3),RESOLVED_ Nullable(DateTime64(3)), " 230 | + " PROCESS_INSTANCE_KEY_ UInt64, ELEMENT_INSTANCE_KEY_ UInt64,JOB_KEY_ UInt64," 231 | + " PROCESS_DEFINITION_KEY_ UInt64)" 232 | + " engine = ReplacingMergeTree(CREATED_) PARTITION BY toYYYYMM(CREATED_) ORDER BY KEY_", 233 | table)); 234 | } 235 | } 236 | 237 | public static void createSignalSubscriptionTable( 238 | final String url, final String user, final String password, final String table) 239 | throws SQLException { 240 | try (final Connection conn = DriverManager.getConnection(url, user, password); 241 | final Statement stmt = conn.createStatement()) { 242 | stmt.execute( 243 | String.format( 244 | "create table if not exists %1$s (ID_ String,SIGNAL_NAME_ String," 245 | + " TIMESTAMP_ DateTime64(3),STATE_ String," 246 | + " CATCH_ELEMENT_INSTANCE_KEY_ Int64,PROCESS_DEFINITION_KEY_ Int64," 247 | + " BPMN_PROCESS_ID_ String,CATCH_EVENT_ID_ String) " 248 | + " engine = ReplacingMergeTree(TIMESTAMP_) PARTITION BY toYYYYMM(TIMESTAMP_) ORDER BY ID_", 249 | table)); 250 | } 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/ErrorImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.value.ErrorRecordValue; 12 | import java.sql.*; 13 | 14 | public class ErrorImporter { 15 | public static void batchErrorInsert( 16 | final String url, 17 | final String user, 18 | final String password, 19 | final String table, 20 | final Record record) 21 | throws SQLException { 22 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 23 | final String sql = 24 | String.format( 25 | "insert into %1$s SETTINGS async_insert=1, wait_for_async_insert=0 select POSITION_, ERROR_EVENT_POSITION_, " 26 | + " TIMESTAMP_, PROCESS_INSTANCE_KEY_, EXCEPTION_MESSAGE_,STACKTRACE_" 27 | + " from input('POSITION_ UInt64,ERROR_EVENT_POSITION_ UInt64," 28 | + " TIMESTAMP_ DateTime64(3), " 29 | + " PROCESS_INSTANCE_KEY_ UInt64,EXCEPTION_MESSAGE_ String," 30 | + " STACKTRACE_ String')", 31 | table); 32 | 33 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 34 | final ErrorRecordValue error = (ErrorRecordValue) record.getValue(); 35 | // 记录Id 36 | ps.setLong(1, record.getPosition()); 37 | // 发生位置 38 | ps.setLong(2, error.getErrorEventPosition()); 39 | // 记录时间 40 | ps.setLong(3, record.getTimestamp()); 41 | // 流程实例Id 42 | ps.setLong(4, error.getProcessInstanceKey()); 43 | // 异常信息 44 | ps.setString(5, error.getExceptionMessage()); 45 | // 堆栈 46 | ps.setString(6, error.getStacktrace()); 47 | ps.addBatch(); 48 | ps.executeBatch(); 49 | } 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/IncidentImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.intent.IncidentIntent; 12 | import io.camunda.zeebe.protocol.record.intent.Intent; 13 | import io.camunda.zeebe.protocol.record.value.IncidentRecordValue; 14 | import java.sql.*; 15 | 16 | public class IncidentImporter { 17 | public static void batchIncidentInsertOrUpdate( 18 | final String url, 19 | final String user, 20 | final String password, 21 | final String table, 22 | final Record record) 23 | throws SQLException { 24 | final Intent intent = record.getIntent(); 25 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 26 | String sql = ""; 27 | 28 | if (intent == IncidentIntent.CREATED) { 29 | sql = 30 | String.format( 31 | "insert into %1$s SETTINGS async_insert=1, wait_for_async_insert=0 select KEY_, BPMN_PROCESS_ID_, ERROR_MSG_, " 32 | + " ERROR_TYPE_, CREATED_, RESOLVED_, PROCESS_INSTANCE_KEY_, " 33 | + " ELEMENT_INSTANCE_KEY_, JOB_KEY_, PROCESS_DEFINITION_KEY_ " 34 | + " from input('KEY_ UInt64,BPMN_PROCESS_ID_ String, ERROR_MSG_ String," 35 | + " ERROR_TYPE_ String," 36 | + " CREATED_ DateTime64(3), RESOLVED_ Nullable(DateTime64(3)), PROCESS_INSTANCE_KEY_ UInt64, " 37 | + " ELEMENT_INSTANCE_KEY_ UInt64, JOB_KEY_ UInt64, PROCESS_DEFINITION_KEY_ UInt64')", 38 | table); 39 | } else if (intent == IncidentIntent.RESOLVED) { 40 | sql = 41 | String.format( 42 | "alter table %1$s update RESOLVED_ = toDateTime64(?/1000,3) where KEY_ = ?", table); 43 | } 44 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 45 | final IncidentRecordValue incident = (IncidentRecordValue) record.getValue(); 46 | if (intent == IncidentIntent.CREATED) { 47 | // 记录Id 48 | ps.setLong(1, record.getKey()); 49 | // 流程定义标识 50 | ps.setString(2, incident.getBpmnProcessId()); 51 | // 节点定义Id 52 | ps.setString(3, incident.getErrorMessage()); 53 | // 任务处理器 54 | ps.setString(4, incident.getErrorType().name()); 55 | // 开始时间 56 | ps.setLong(5, record.getTimestamp()); 57 | // 最后更新时间 58 | ps.setString(6, null); 59 | // 版本号 60 | ps.setLong(7, incident.getProcessInstanceKey()); 61 | // 节点实例Id 62 | ps.setLong(8, incident.getElementInstanceKey()); 63 | // 节点实例Id 64 | ps.setLong(9, incident.getJobKey()); 65 | // 流程定义Id 66 | ps.setLong(10, incident.getProcessDefinitionKey()); 67 | 68 | ps.addBatch(); 69 | ps.executeBatch(); 70 | } else if (intent == IncidentIntent.RESOLVED) { 71 | // 更新时间 72 | ps.setLong(1, record.getTimestamp()); 73 | // 记录Id 74 | ps.setLong(2, record.getKey()); 75 | 76 | System.out.println("incident----->record.getTimestamp()->" + record.getTimestamp()); 77 | ps.execute(); 78 | } 79 | } 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/JobImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.intent.Intent; 12 | import io.camunda.zeebe.protocol.record.intent.JobIntent; 13 | import io.camunda.zeebe.protocol.record.value.JobRecordValue; 14 | import java.sql.*; 15 | 16 | public class JobImporter { 17 | public static void batchJobInsertOrUpdate( 18 | final String url, 19 | final String user, 20 | final String password, 21 | final String table, 22 | final Record record) 23 | throws SQLException { 24 | final Intent intent = record.getIntent(); 25 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 26 | final String sql; 27 | if (intent == JobIntent.CREATED) { 28 | sql = 29 | String.format( 30 | "insert into %1$s SETTINGS async_insert=1, wait_for_async_insert=0 select KEY_, BPMN_PROCESS_ID_, " 31 | + " ELEMENT_ID_, WORKER_, JOB_TYPE_, STATE_, RETRIES_, START_, END_, PROCESS_INSTANCE_KEY_, " 32 | + " ELEMENT_INSTANCE_KEY_, PROCESS_DEFINITION_KEY_ " 33 | + " from input('KEY_ UInt64,BPMN_PROCESS_ID_ String, ELEMENT_ID_ String," 34 | + " WORKER_ String,JOB_TYPE_ String,STATE_ String, RETRIES_ UInt8," 35 | + " START_ DateTime64(3), END_ Nullable(DateTime64(3)), PROCESS_INSTANCE_KEY_ UInt64, " 36 | + " ELEMENT_INSTANCE_KEY_ UInt64, PROCESS_DEFINITION_KEY_ UInt64')", 37 | table); 38 | } else { 39 | sql = 40 | String.format( 41 | "alter table %1$s update END_ = toDateTime64(?/1000,3), " 42 | + " WORKER_ =?, RETRIES_=?, STATE_=? " 43 | + " where KEY_ = ?", 44 | table); 45 | } 46 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 47 | final JobRecordValue job = (JobRecordValue) record.getValue(); 48 | if (intent == JobIntent.CREATED) { 49 | // 记录Id 50 | ps.setLong(1, record.getKey()); 51 | // 流程定义标识 52 | ps.setString(2, job.getBpmnProcessId()); 53 | // 节点定义Id 54 | ps.setString(3, job.getElementId()); 55 | // 任务处理器 56 | ps.setString(4, job.getWorker()); 57 | // 任务类型 58 | ps.setString(5, job.getType()); 59 | // 状态 60 | ps.setString(6, record.getIntent().name().toLowerCase()); 61 | // 重试次数 62 | ps.setInt(7, job.getRetries()); 63 | // 开始时间 64 | ps.setLong(8, record.getTimestamp()); 65 | // 最后更新时间 66 | ps.setString(9, null); 67 | // 版本号 68 | ps.setLong(10, job.getProcessInstanceKey()); 69 | // 节点实例Id 70 | ps.setLong(11, job.getElementInstanceKey()); 71 | // 流程定义Id 72 | ps.setLong(12, job.getProcessDefinitionKey()); 73 | 74 | ps.addBatch(); 75 | ps.executeBatch(); 76 | } else { 77 | // 更新时间 78 | ps.setLong(1, record.getTimestamp()); 79 | // 任务处理器 80 | ps.setString(2, job.getWorker()); 81 | // 重试次数 82 | ps.setInt(3, job.getRetries()); 83 | // 状态 84 | ps.setString(4, record.getIntent().name().toLowerCase()); 85 | // 记录Id 86 | ps.setLong(5, record.getKey()); 87 | 88 | ps.execute(); 89 | } 90 | } 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/MessageImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.value.MessageRecordValue; 12 | import java.sql.*; 13 | 14 | public class MessageImporter { 15 | public static void batchMessageInsert( 16 | final String url, 17 | final String user, 18 | final String password, 19 | final String table, 20 | final Record record) 21 | throws SQLException { 22 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 23 | final String sql = 24 | String.format( 25 | "insert into %1$s SETTINGS async_insert=1, wait_for_async_insert=0 select KEY_, NAME_, " 26 | + " TIMESTAMP_, STATE_, CORRELATION_KEY_,MESSAGE_ID_,PAYLOAD_" 27 | + " from input('KEY_ UInt64,NAME_ String," 28 | + " TIMESTAMP_ DateTime64(3), STATE_ String," 29 | + " CORRELATION_KEY_ String,MESSAGE_ID_ String," 30 | + " PAYLOAD_ String')", 31 | table); 32 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 33 | final MessageRecordValue msg = (MessageRecordValue) record.getValue(); 34 | // 记录Id 35 | ps.setLong(1, record.getKey()); 36 | // 名称 37 | ps.setString(2, msg.getName()); 38 | // 记录时间 39 | ps.setLong(3, record.getTimestamp()); 40 | // 状态 41 | ps.setString(4, record.getIntent().name().toLowerCase()); 42 | // 关联key 43 | ps.setString(5, msg.getCorrelationKey()); 44 | // 消息id 45 | ps.setString(6, msg.getMessageId()); 46 | // 报文 47 | ps.setString(7, msg.getVariables().toString()); 48 | ps.addBatch(); 49 | ps.executeBatch(); 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/MessageSubscriptionImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.value.MessageStartEventSubscriptionRecordValue; 12 | import io.camunda.zeebe.protocol.record.value.MessageSubscriptionRecordValue; 13 | import java.sql.*; 14 | import java.util.UUID; 15 | 16 | public class MessageSubscriptionImporter { 17 | public static void batchMessageSubscriptionInsert( 18 | final String url, 19 | final String user, 20 | final String password, 21 | final String table, 22 | final Record record) 23 | throws SQLException { 24 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 25 | final String sql = 26 | String.format( 27 | "insert into %1$s SETTINGS async_insert=1, wait_for_async_insert=0 select ID_, MESSAGE_NAME_, MESSAGE_KEY_," 28 | + " TIMESTAMP_, STATE_, PROCESS_INSTANCE_KEY_,ELEMENT_INSTANCE_KEY_," 29 | + " PROCESS_DEFINITION_KEY_,CORRELATION_KEY_,TARGET_FLOW_NODE_ID_" 30 | + " from input('ID_ String,MESSAGE_NAME_ String,MESSAGE_KEY_ Int64," 31 | + " TIMESTAMP_ DateTime64(3), STATE_ String," 32 | + " PROCESS_INSTANCE_KEY_ Int64,ELEMENT_INSTANCE_KEY_ Int64,PROCESS_DEFINITION_KEY_ Int64," 33 | + " CORRELATION_KEY_ Nullable(String),TARGET_FLOW_NODE_ID_ Nullable(String)')", 34 | table); 35 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 36 | final MessageSubscriptionRecordValue msg = 37 | (MessageSubscriptionRecordValue) record.getValue(); 38 | // 记录Id 39 | ps.setString(1, generateId()); 40 | // 名称 41 | ps.setString(2, msg.getMessageName()); 42 | ps.setLong(3, -1L); 43 | // 记录时间 44 | ps.setLong(4, record.getTimestamp()); 45 | // 状态 46 | ps.setString(5, record.getIntent().name().toLowerCase()); 47 | ps.setLong(6, msg.getProcessInstanceKey()); 48 | ps.setLong(7, msg.getElementInstanceKey()); 49 | 50 | ps.setLong(8, -1L); 51 | // 关联key 52 | ps.setString(9, msg.getCorrelationKey()); 53 | 54 | ps.setString(10, ""); 55 | ps.addBatch(); 56 | ps.executeBatch(); 57 | } 58 | } 59 | } 60 | 61 | public static void batchMessageStartEventSubscriptionInsert( 62 | final String url, 63 | final String user, 64 | final String password, 65 | final String table, 66 | final Record record) 67 | throws SQLException { 68 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 69 | final String sql = 70 | String.format( 71 | "insert into %1$s select ID_, MESSAGE_NAME_,MESSAGE_KEY_," 72 | + " TIMESTAMP_, STATE_, PROCESS_INSTANCE_KEY_,ELEMENT_INSTANCE_KEY_," 73 | + " PROCESS_DEFINITION_KEY_,CORRELATION_KEY_,TARGET_FLOW_NODE_ID_" 74 | + " from input('ID_ String,MESSAGE_NAME_ String,MESSAGE_KEY_ Int64," 75 | + " TIMESTAMP_ DateTime64(3), STATE_ String," 76 | + " PROCESS_INSTANCE_KEY_ Int64,ELEMENT_INSTANCE_KEY_ Int64,PROCESS_DEFINITION_KEY_ Int64," 77 | + " CORRELATION_KEY_ Nullable(String),TARGET_FLOW_NODE_ID_ Nullable(String)')", 78 | table); 79 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 80 | final MessageStartEventSubscriptionRecordValue msg = 81 | (MessageStartEventSubscriptionRecordValue) record.getValue(); 82 | // 记录Id 83 | ps.setString(1, generateId()); 84 | // 消息名称 85 | ps.setString(2, msg.getMessageName()); 86 | // 消息实例Key 87 | ps.setLong(3, msg.getMessageKey()); 88 | // 记录时间 89 | ps.setLong(4, record.getTimestamp()); 90 | // 状态 91 | ps.setString(5, record.getIntent().name().toLowerCase()); 92 | ps.setLong(6, msg.getProcessInstanceKey()); 93 | ps.setLong(7, -1); 94 | 95 | ps.setLong(8, msg.getProcessDefinitionKey()); 96 | // 关联key 97 | ps.setString(9, msg.getCorrelationKey()); 98 | 99 | ps.setString(10, msg.getStartEventId()); 100 | ps.addBatch(); 101 | ps.executeBatch(); 102 | } 103 | } 104 | } 105 | 106 | private static String generateId() { 107 | return UUID.randomUUID().toString(); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/ProcessImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.value.deployment.Process; 12 | import java.nio.charset.StandardCharsets; 13 | import java.sql.*; 14 | 15 | public class ProcessImporter { 16 | public static void batchProcessInsert( 17 | final String url, 18 | final String user, 19 | final String password, 20 | final String table, 21 | final Record record) 22 | throws SQLException { 23 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 24 | final String sql = 25 | String.format( 26 | "insert into %1$s SETTINGS async_insert=1, wait_for_async_insert=0 select KEY_," 27 | + " BPMN_PROCESS_ID_, RESOURCE_NAME_, " 28 | + " TIMESTAMP_, PARTITION_ID_, POSITION_, VERSION_, RESOURCE_" 29 | + " from input('KEY_ UInt64, BPMN_PROCESS_ID_ String, RESOURCE_NAME_ String," 30 | + " TIMESTAMP_ DateTime64(3), PARTITION_ID_ UInt16, POSITION_ UInt64,VERSION_ UInt16,RESOURCE_ String')", 31 | table); 32 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 33 | final Process process = (Process) record.getValue(); 34 | // 流程定义Id 35 | ps.setLong(1, record.getKey()); 36 | // 流程定义键值 37 | ps.setString(2, process.getBpmnProcessId()); 38 | // 流程定义文件名 39 | ps.setString(3, process.getResourceName()); 40 | // 部署时间 41 | ps.setLong(4, record.getTimestamp()); 42 | // 所属分区 43 | ps.setInt(5, record.getPartitionId()); 44 | // 位置 45 | ps.setLong(6, record.getPosition()); 46 | // 版本号 47 | ps.setInt(7, process.getVersion()); 48 | // 模型定义信息 49 | final String resource = new String(process.getResource(), StandardCharsets.UTF_8); 50 | ps.setString(8, resource); 51 | 52 | ps.addBatch(); 53 | ps.executeBatch(); 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/ProcessInstanceImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.intent.Intent; 12 | import io.camunda.zeebe.protocol.record.intent.ProcessInstanceIntent; 13 | import io.camunda.zeebe.protocol.record.value.BpmnElementType; 14 | import io.camunda.zeebe.protocol.record.value.ProcessInstanceRecordValue; 15 | import java.sql.*; 16 | 17 | public class ProcessInstanceImporter { 18 | 19 | public static void batchProcessInstanceInsertOrUpdate( 20 | final String url, 21 | final String user, 22 | final String password, 23 | final String table, 24 | final Record record) 25 | throws SQLException { 26 | final Intent intent = record.getIntent(); 27 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 28 | final String sql; 29 | if (intent == ProcessInstanceIntent.ELEMENT_ACTIVATED) { 30 | sql = 31 | String.format( 32 | "insert into %1$s SETTINGS async_insert=1, wait_for_async_insert=0 select KEY_, BPMN_PROCESS_ID_, " 33 | + " PROCESS_DEFINITION_KEY_, START_,END_, PARTITION_ID_, VERSION_, STATE_," 34 | + " PARENT_PROCESS_INSTANCE_KEY_, PARENT_ELEMENT_INSTANCE_KEY_" 35 | + " from input('KEY_ UInt64,BPMN_PROCESS_ID_ String," 36 | + " PROCESS_DEFINITION_KEY_ UInt64, START_ DateTime64(3), END_ Nullable(DateTime64(3))," 37 | + " PARTITION_ID_ UInt16, VERSION_ UInt16,STATE_ String, " 38 | + " PARENT_PROCESS_INSTANCE_KEY_ Int64, PARENT_ELEMENT_INSTANCE_KEY_ Int64')", 39 | table); 40 | } else { 41 | sql = 42 | String.format( 43 | "alter table %1$s update END_ = toDateTime64(?/1000,3),STATE_ =? where KEY_ = ?", 44 | table); 45 | } 46 | 47 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 48 | final ProcessInstanceRecordValue processInstance = 49 | (ProcessInstanceRecordValue) record.getValue(); 50 | if (BpmnElementType.PROCESS.name().equals(processInstance.getBpmnElementType().name())) { 51 | if (intent == ProcessInstanceIntent.ELEMENT_ACTIVATED) { 52 | // 流程实例Id 53 | ps.setLong(1, record.getKey()); 54 | // 流程定义键值 55 | ps.setString(2, processInstance.getBpmnProcessId()); 56 | // 流程定义Id 57 | ps.setLong(3, processInstance.getProcessDefinitionKey()); 58 | // 创建时间 59 | ps.setLong(4, record.getTimestamp()); 60 | ps.setString(5, null); 61 | // 所属分区 62 | ps.setInt(6, record.getPartitionId()); 63 | // 版本号 64 | ps.setInt(7, processInstance.getVersion()); 65 | // 状态 66 | ps.setString(8, "Active"); 67 | // 父流程实例Id 68 | ps.setLong(9, processInstance.getParentProcessInstanceKey()); 69 | // 父节点实例Id 70 | ps.setLong(10, processInstance.getParentElementInstanceKey()); 71 | ps.addBatch(); 72 | ps.executeBatch(); 73 | } else if (intent == ProcessInstanceIntent.ELEMENT_COMPLETED) { 74 | // 结束时间 75 | ps.setLong(1, record.getTimestamp()); 76 | // 状态 77 | ps.setString(2, "Completed"); 78 | // 流程实例Id 79 | ps.setLong(3, record.getKey()); 80 | // System.out.println("record.getTimestamp()->"+record.getTimestamp()); 81 | ps.execute(); 82 | } else if (intent == ProcessInstanceIntent.ELEMENT_TERMINATED) { 83 | // 结束时间 84 | ps.setLong(1, record.getTimestamp()); 85 | // 状态 86 | ps.setString(2, "Terminated"); 87 | // 流程实例Id 88 | ps.setLong(3, record.getKey()); 89 | 90 | ps.execute(); 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | public static void batchElementInstanceInsert( 98 | final String url, 99 | final String user, 100 | final String password, 101 | final String table, 102 | final Record record) 103 | throws SQLException { 104 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 105 | final String sql = 106 | String.format( 107 | "insert into %1$s select ID,KEY_, BPMN_PROCESS_ID_, PROCESS_DEFINITION_KEY_, " 108 | + " TIMESTAMP_,INTENT_ ,PARTITION_ID_," 109 | + " POSITION_, PROCESS_INSTANCE_KEY_," 110 | + " FLOW_SCOPE_KEY_,ELEMENT_ID_,BPMN_ELEMENT_TYPE_" 111 | + " from input('ID String,KEY_ UInt64,BPMN_PROCESS_ID_ String," 112 | + " PROCESS_DEFINITION_KEY_ UInt64, TIMESTAMP_ DateTime64(3), " 113 | + " INTENT_ String,PARTITION_ID_ UInt16," 114 | + " POSITION_ UInt64, PROCESS_INSTANCE_KEY_ UInt64," 115 | + " FLOW_SCOPE_KEY_ Int64,ELEMENT_ID_ String,BPMN_ELEMENT_TYPE_ String')", 116 | table); 117 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 118 | 119 | final ProcessInstanceRecordValue processInstance = 120 | (ProcessInstanceRecordValue) record.getValue(); 121 | // 记录Id 122 | ps.setString(1, record.getPartitionId() + "-" + record.getPosition()); 123 | // 元素记录Id 124 | ps.setLong(2, record.getKey()); 125 | // 流程定义键值 126 | ps.setString(3, processInstance.getBpmnProcessId()); 127 | // 流程定义Id 128 | ps.setLong(4, processInstance.getProcessDefinitionKey()); 129 | // 创建时间 130 | ps.setLong(5, record.getTimestamp()); 131 | // 活动状态 132 | ps.setString(6, record.getIntent().name()); 133 | // 所属分区 134 | ps.setInt(7, record.getPartitionId()); 135 | // 版本号 136 | ps.setLong(8, record.getPosition()); 137 | // 流程实例Id 138 | ps.setLong(9, processInstance.getProcessInstanceKey()); 139 | // 范围Id 140 | ps.setLong(10, processInstance.getFlowScopeKey()); 141 | // 元素定义Id 142 | ps.setString(11, processInstance.getElementId()); 143 | // 元素定义类型 144 | ps.setString(12, processInstance.getBpmnElementType().name()); 145 | ps.addBatch(); 146 | ps.executeBatch(); 147 | } catch (final SQLException e) { 148 | e.printStackTrace(); 149 | } 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/SignalSubscriptionImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.value.SignalSubscriptionRecordValue; 12 | import java.sql.*; 13 | import java.util.UUID; 14 | 15 | public class SignalSubscriptionImporter { 16 | 17 | public static void batchSignalSubscriptionInsert( 18 | final String url, 19 | final String user, 20 | final String password, 21 | final String table, 22 | final Record record) 23 | throws SQLException { 24 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 25 | final String sql = 26 | String.format( 27 | "insert into %1$s select ID_, SIGNAL_NAME_," 28 | + " TIMESTAMP_, STATE_, CATCH_ELEMENT_INSTANCE_KEY_," 29 | + " PROCESS_DEFINITION_KEY_,BPMN_PROCESS_ID_,CATCH_EVENT_ID_" 30 | + " from input('ID_ String,SIGNAL_NAME_ String," 31 | + " TIMESTAMP_ DateTime64(3), STATE_ String," 32 | + " CATCH_ELEMENT_INSTANCE_KEY_ Int64,PROCESS_DEFINITION_KEY_ Int64," 33 | + " BPMN_PROCESS_ID_ String,CATCH_EVENT_ID_ String')", 34 | table); 35 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 36 | final SignalSubscriptionRecordValue signal = 37 | (SignalSubscriptionRecordValue) record.getValue(); 38 | // 记录Id 39 | ps.setString(1, generateId()); 40 | // 信号名称 41 | ps.setString(2, signal.getSignalName()); 42 | // 记录时间 43 | ps.setLong(3, record.getTimestamp()); 44 | // 状态 45 | ps.setString(4, record.getIntent().name().toLowerCase()); 46 | ps.setLong(5, signal.getCatchEventInstanceKey()); 47 | 48 | ps.setLong(6, signal.getProcessDefinitionKey()); 49 | // 关联key 50 | ps.setString(7, signal.getBpmnProcessId()); 51 | 52 | ps.setString(8, signal.getCatchEventId()); 53 | ps.addBatch(); 54 | ps.executeBatch(); 55 | } 56 | } 57 | } 58 | 59 | private static String generateId() { 60 | return UUID.randomUUID().toString(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/TimerImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.value.TimerRecordValue; 12 | import java.sql.*; 13 | 14 | public class TimerImporter { 15 | public static void batchTimerInsert( 16 | final String url, 17 | final String user, 18 | final String password, 19 | final String table, 20 | final Record record) 21 | throws SQLException { 22 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 23 | final String sql = 24 | String.format( 25 | "insert into %1$s SETTINGS async_insert=1, wait_for_async_insert=0 select KEY_, STATE_, REPETITIONS, " 26 | + " TIMESTAMP_, DUE_DATE_, PROCESS_INSTANCE_KEY_, PROCESS_DEFINITION_KEY_,ELEMENT_INSTANCE_KEY_, TARGET_ELEMENT_ID_" 27 | + " from input('KEY_ UInt64,STATE_ String, REPETITIONS Int16," 28 | + " TIMESTAMP_ DateTime64(3), DUE_DATE_ DateTime64(3), " 29 | + " PROCESS_INSTANCE_KEY_ Int64,PROCESS_DEFINITION_KEY_ UInt64," 30 | + " ELEMENT_INSTANCE_KEY_ Int64, TARGET_ELEMENT_ID_ String')", 31 | table); 32 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 33 | final TimerRecordValue timer = (TimerRecordValue) record.getValue(); 34 | // 记录Id 35 | ps.setLong(1, record.getKey()); 36 | // 状态 37 | ps.setString(2, record.getIntent().name().toLowerCase()); 38 | // 重复次数 39 | ps.setInt(3, timer.getRepetitions()); 40 | // 记录时间 41 | ps.setLong(4, record.getTimestamp()); 42 | // 截止时间 43 | ps.setLong(5, timer.getDueDate()); 44 | // 流程实例Id 45 | ps.setLong(6, timer.getProcessInstanceKey()); 46 | // 流程定义id 47 | ps.setLong(7, timer.getProcessDefinitionKey()); 48 | // 节点实例Id 49 | ps.setLong(8, timer.getElementInstanceKey()); 50 | // 节点Id 51 | ps.setString(9, timer.getTargetElementId()); 52 | 53 | ps.addBatch(); 54 | ps.executeBatch(); 55 | 56 | } catch (final Exception e) { 57 | e.printStackTrace(); 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/io/zeebe/clickhouse/exporter/importer/VariableImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter.importer; 9 | 10 | import io.camunda.zeebe.protocol.record.Record; 11 | import io.camunda.zeebe.protocol.record.value.VariableRecordValue; 12 | import java.sql.*; 13 | 14 | public class VariableImporter { 15 | 16 | public static void batchVariableInsert( 17 | final String url, 18 | final String user, 19 | final String password, 20 | final String table, 21 | final Record record) 22 | throws SQLException { 23 | try (final Connection conn = DriverManager.getConnection(url, user, password)) { 24 | 25 | final String sql = 26 | String.format( 27 | "insert into %1$s SETTINGS async_insert=1, wait_for_async_insert=0 select ID, NAME_, VALUE_, " 28 | + " TIMESTAMP_, PARTITION_ID_, POSITION_, " 29 | + " PROCESS_INSTANCE_KEY_, PROCESS_DEFINITION_KEY_,SCOPE_KEY_, STATE_" 30 | + " from input('ID String,NAME_ String, VALUE_ String," 31 | + " TIMESTAMP_ DateTime64(3), PARTITION_ID_ UInt16, POSITION_ UInt64," 32 | + " PROCESS_INSTANCE_KEY_ UInt64,PROCESS_DEFINITION_KEY_ UInt64," 33 | + " SCOPE_KEY_ Int64, STATE_ String')", 34 | table); 35 | try (final PreparedStatement ps = conn.prepareStatement(sql)) { 36 | final VariableRecordValue variable = (VariableRecordValue) record.getValue(); 37 | // 记录Id 38 | ps.setString(1, String.format("%s-%s", record.getPartitionId(), record.getPosition())); 39 | // ps.setString(1,generateId()); 40 | // 变量名 41 | ps.setString(2, variable.getName()); 42 | // 变量值 43 | ps.setString(3, variable.getValue()); 44 | // 记录时间 45 | ps.setLong(4, record.getTimestamp()); 46 | // 所属分区 47 | ps.setInt(5, record.getPartitionId()); 48 | // 位置 49 | ps.setLong(6, record.getPosition()); 50 | // 流程实例Id 51 | ps.setLong(7, variable.getProcessInstanceKey()); 52 | // 流程定义id 53 | ps.setLong(8, variable.getProcessDefinitionKey()); 54 | // 所属范围 55 | ps.setLong(9, variable.getScopeKey()); 56 | // 状态 57 | ps.setString(10, record.getIntent().name().toLowerCase()); 58 | 59 | ps.addBatch(); 60 | ps.executeBatch(); 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/io/zeebe/clickhouse/exporter/ClickHouseExporterIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under 3 | * one or more contributor license agreements. See the NOTICE file distributed 4 | * with this work for additional information regarding copyright ownership. 5 | * Licensed under the Zeebe Community License 1.1. You may not use this file 6 | * except in compliance with the Zeebe Community License 1.1. 7 | */ 8 | package io.zeebe.clickhouse.exporter; 9 | 10 | import static org.assertj.core.api.Assertions.assertThat; 11 | 12 | import com.clickhouse.data.ClickHouseDataType; 13 | import io.camunda.zeebe.client.ZeebeClient; 14 | import io.camunda.zeebe.client.api.response.DeploymentEvent; 15 | import io.camunda.zeebe.model.bpmn.Bpmn; 16 | import io.camunda.zeebe.model.bpmn.BpmnModelInstance; 17 | import io.zeebe.containers.ZeebeContainer; 18 | import java.sql.*; 19 | import java.util.concurrent.TimeUnit; 20 | import org.agrona.CloseHelper; 21 | import org.junit.Test; 22 | import org.junit.jupiter.api.AfterEach; 23 | import org.junit.jupiter.api.Timeout; 24 | import org.junit.jupiter.api.parallel.Execution; 25 | import org.junit.jupiter.api.parallel.ExecutionMode; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | import org.testcontainers.clickhouse.ClickHouseContainer; 29 | import org.testcontainers.containers.Network; 30 | import org.testcontainers.containers.output.Slf4jLogConsumer; 31 | import org.testcontainers.junit.jupiter.Container; 32 | import org.testcontainers.junit.jupiter.Testcontainers; 33 | import org.testcontainers.utility.DockerImageName; 34 | import org.testcontainers.utility.MountableFile; 35 | 36 | @Testcontainers 37 | @Timeout(value = 2, unit = TimeUnit.MINUTES) 38 | @Execution(ExecutionMode.CONCURRENT) 39 | public class ClickHouseExporterIT { 40 | 41 | private static final String JOB_TYPE = "work"; 42 | private static final String MESSAGE_NAME = "catch"; 43 | private static final String PROCESS_NAME = "testProcess"; 44 | private static final String PROCESS_FILE_NAME = "sample_workflow.bpmn"; 45 | private static final String TASK_NAME = "task"; 46 | private static final BpmnModelInstance SAMPLE_PROCESS = 47 | Bpmn.createExecutableProcess(PROCESS_NAME) 48 | .startEvent() 49 | .intermediateCatchEvent( 50 | "message", 51 | e -> e.message(m -> m.name(MESSAGE_NAME).zeebeCorrelationKeyExpression("orderId"))) 52 | .serviceTask(TASK_NAME, t -> t.zeebeJobType(JOB_TYPE).zeebeTaskHeader("foo", "bar")) 53 | .endEvent() 54 | .done(); 55 | private final MountableFile exporterJar = 56 | MountableFile.forClasspathResource("zeebe-clickhouse-exporter.jar", 775); 57 | private final MountableFile exporterConfig = 58 | MountableFile.forClasspathResource("application.yaml", 775); 59 | private final MountableFile loggingConfig = 60 | MountableFile.forClasspathResource("logback-test.xml", 775); 61 | private final Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(newContainerLogger(), true); 62 | private final Network network = Network.newNetwork(); 63 | 64 | @Container 65 | public ClickHouseContainer clickHouse = 66 | new ClickHouseContainer( 67 | DockerImageName.parse("clickhouse/clickhouse-server:22.8.15.23-alpine")) 68 | .withNetwork(network) 69 | .withNetworkAliases("clickhouse") 70 | .withExposedPorts(8123); 71 | 72 | @Container 73 | public ZeebeContainer zeebeContainer = 74 | new ZeebeContainer(DockerImageName.parse("camunda/zeebe:8.2.0")) 75 | .withNetwork(network) 76 | .withNetworkAliases("zeebe") 77 | .withEnv("ZEEBE_BROKER_NETWORK_ADVERTISEDHOST", "zeebe") 78 | .withEnv("ZEEBE_CLICKHOUSE_URL", "jdbc:ch://clickhouse/default") 79 | .withEnv("ZEEBE_CLICKHOUSE_USER", "default") 80 | .withEnv("ZEEBE_CLICKHOUSE_PASSWORD", "") 81 | .withEnv("ZEEBE_LOG_LEVEL", "debug") 82 | .withCopyFileToContainer( 83 | exporterJar, "/usr/local/zeebe/exporters/zeebe-clickhouse-exporter.jar") 84 | .withCopyFileToContainer(exporterConfig, "/usr/local/zeebe/config/application.yaml") 85 | .withCopyFileToContainer(loggingConfig, "/usr/local/zeebe/config/logback-test.xml") 86 | .withEnv( 87 | "SPRING_CONFIG_ADDITIONAL_LOCATION", "file:/usr/local/zeebe/config/application.yaml") 88 | .withLogConsumer(logConsumer); 89 | 90 | private ZeebeClient zeebeClient; 91 | 92 | private static Logger newContainerLogger() { 93 | return LoggerFactory.getLogger(ClickHouseExporterIT.class.getName() + "." + "zeebeContainer"); 94 | } 95 | 96 | @AfterEach 97 | void tearDown() { 98 | CloseHelper.quietCloseAll(zeebeClient, zeebeContainer, clickHouse, network); 99 | } 100 | 101 | @Test 102 | public void testClickHouseContainer() throws SQLException { 103 | clickHouse.start(); 104 | 105 | final String sql = "select cast(1 as UInt64) a, cast([1, 2] as Array(Int8)) b"; 106 | try (final Connection conn = 107 | DriverManager.getConnection( 108 | clickHouse.getJdbcUrl(), clickHouse.getUsername(), clickHouse.getPassword()); 109 | final Statement stmt = conn.createStatement(); 110 | final ResultSet rs = stmt.executeQuery(sql)) { 111 | 112 | assertThat(rs.next()).isTrue(); 113 | assertThat(rs.getMetaData().getColumnTypeName(1)).isEqualTo(ClickHouseDataType.UInt64.name()); 114 | assertThat(rs.getMetaData().getColumnTypeName(2)).isEqualTo("Array(Int8)"); 115 | assertThat(rs.next()).isFalse(); 116 | } 117 | } 118 | 119 | @Test 120 | public void shouldExportProcessToClickHouse() throws SQLException, InterruptedException { 121 | // given 122 | clickHouse.start(); 123 | zeebeContainer.start(); 124 | zeebeClient = getLazyZeebeClient(); 125 | 126 | // when 127 | final DeploymentEvent deploymentEvent = 128 | zeebeClient 129 | .newDeployResourceCommand() 130 | .addProcessModel(SAMPLE_PROCESS, PROCESS_FILE_NAME) 131 | .send() 132 | .join(); 133 | 134 | Thread.sleep(1000); 135 | // then 136 | final String sql = 137 | "select KEY_,BPMN_PROCESS_ID_,RESOURCE_NAME_,TIMESTAMP_,PARTITION_ID_,POSITION_,VERSION_,RESOURCE_ from PROCESS"; 138 | 139 | try (final Connection conn = 140 | DriverManager.getConnection( 141 | clickHouse.getJdbcUrl(), clickHouse.getUsername(), clickHouse.getPassword()); 142 | final Statement stmt = conn.createStatement(); 143 | final ResultSet rs = stmt.executeQuery(sql)) { 144 | 145 | assertThat(rs.next()).isTrue(); 146 | assertThat(rs.getMetaData().getColumnTypeName(1)).isEqualTo(ClickHouseDataType.UInt64.name()); 147 | assertThat(rs.getMetaData().getColumnTypeName(2)).isEqualTo(ClickHouseDataType.String.name()); 148 | assertThat(rs.getMetaData().getColumnTypeName(3)).isEqualTo(ClickHouseDataType.String.name()); 149 | assertThat(rs.getMetaData().getColumnTypeName(4)).isEqualTo("DateTime64(3)"); 150 | assertThat(rs.getMetaData().getColumnTypeName(5)).isEqualTo(ClickHouseDataType.UInt16.name()); 151 | assertThat(rs.getMetaData().getColumnTypeName(6)).isEqualTo(ClickHouseDataType.UInt64.name()); 152 | assertThat(rs.getMetaData().getColumnTypeName(7)).isEqualTo(ClickHouseDataType.UInt16.name()); 153 | assertThat(rs.getMetaData().getColumnTypeName(8)).isEqualTo(ClickHouseDataType.String.name()); 154 | 155 | assertThat(rs.getLong(1)) 156 | .isEqualTo(deploymentEvent.getProcesses().get(0).getProcessDefinitionKey()); 157 | assertThat(rs.getString(2)).isEqualTo(PROCESS_NAME); 158 | assertThat(rs.getString(3)).isEqualTo(PROCESS_FILE_NAME); 159 | assertThat(rs.getInt(7)).isEqualTo(deploymentEvent.getProcesses().get(0).getVersion()); 160 | assertThat(rs.getString(8)).isEqualTo(Bpmn.convertToString(SAMPLE_PROCESS)); 161 | 162 | assertThat(rs.next()).isFalse(); 163 | } 164 | } 165 | 166 | private ZeebeClient getLazyZeebeClient() { 167 | if (zeebeClient == null) { 168 | zeebeClient = 169 | ZeebeClient.newClientBuilder() 170 | .gatewayAddress(zeebeContainer.getExternalGatewayAddress()) 171 | .usePlaintext() 172 | .build(); 173 | } 174 | return zeebeClient; 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/test/resources/application.yaml: -------------------------------------------------------------------------------- 1 | zeebe: 2 | broker: 3 | exporters: 4 | clickhouse: 5 | className: io.zeebe.clickhouse.exporter.ClickHouseExporter 6 | jarPath: exporters/zeebe-clickhouse-exporter.jar 7 | -------------------------------------------------------------------------------- /src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | --------------------------------------------------------------------------------