├── .github └── ISSUE_TEMPLATE │ └── 10_unsupported.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── docs ├── en │ └── operations │ │ └── utilities │ │ └── clickhouse-copier.md ├── ru │ └── operations │ │ └── utilities │ │ └── clickhouse-copier.md └── zh │ └── operations │ └── utilities │ └── clickhouse-copier.md ├── programs └── copier │ ├── Aliases.h │ ├── CMakeLists.txt │ ├── ClusterCopier.cpp │ ├── ClusterCopier.h │ ├── ClusterCopierApp.cpp │ ├── ClusterCopierApp.h │ ├── ClusterPartition.h │ ├── Internals.cpp │ ├── Internals.h │ ├── ShardPartition.cpp │ ├── ShardPartition.h │ ├── ShardPartitionPiece.cpp │ ├── ShardPartitionPiece.h │ ├── StatusAccumulator.cpp │ ├── StatusAccumulator.h │ ├── TaskCluster.cpp │ ├── TaskCluster.h │ ├── TaskShard.cpp │ ├── TaskShard.h │ ├── TaskTable.cpp │ ├── TaskTable.h │ ├── ZooKeeperStaff.h │ └── clickhouse-copier.cpp └── tests └── integration └── test_cluster_copier ├── __init__.py ├── configs ├── conf.d │ ├── clusters.xml │ ├── clusters_trivial.xml │ ├── ddl.xml │ └── query_log.xml ├── config-copier.xml └── users.xml ├── configs_three_nodes ├── conf.d │ ├── clusters.xml │ └── ddl.xml ├── config-copier.xml └── users.xml ├── configs_two_nodes ├── conf.d │ ├── clusters.xml │ ├── ddl.xml │ └── storage_configuration.xml ├── config-copier.xml └── users.xml ├── task0_description.xml ├── task_drop_target_partition.xml ├── task_month_to_week_description.xml ├── task_no_arg.xml ├── task_no_index.xml ├── task_non_partitioned_table.xml ├── task_self_copy.xml ├── task_skip_index.xml ├── task_taxi_data.xml ├── task_test_block_size.xml ├── task_trivial.xml ├── task_trivial_without_arguments.xml ├── task_ttl_columns.xml ├── task_ttl_move_to_volume.xml ├── task_with_different_schema.xml ├── test.py ├── test_three_nodes.py ├── test_trivial.py └── test_two_nodes.py /.github/ISSUE_TEMPLATE/10_unsupported.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Express gratitude for this legacy tool 3 | about: Express gratitude for this legacy tool, which is no longer supported 4 | title: '' 5 | labels: wontfix 6 | assignees: '' 7 | --- 8 | 9 | > This tool is no longer supported, but asking a question or reporting and observation won't harm. 10 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # ClickHouse Community Code of Conduct 2 | This code of conduct governs ClickHouse Community discussions, channels, and events. 3 | 4 | ## Introduction 5 | * Diversity and inclusion make our community strong. We encourage, and desire, participation from the most varied and diverse backgrounds possible. 6 | * This isn’t an exhaustive list of things that you can’t do. Rather, take it in the spirit in which it’s intended - a guide to make it easier to build, and sustain, the diverse technical communities in which we participate. 7 | * Our goal is to maintain a safe and friendly community for everyone, regardless of experience, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, nationality, or other defining characteristic. 8 | * This code and related procedures apply to unacceptable behavior occurring in all community venues, including behavior outside the scope of community activities — online and in-person — as well as in all one-on-one communications, and anywhere such behavior has the potential to adversely affect the safety and well-being of community members. 9 | 10 | ## Expected Behavior 11 | * Be welcoming. 12 | * Be kind and patient. 13 | * Look out for each other. 14 | * Be careful and aware of your wording and tone. 15 | 16 | ## Unacceptable Behavior 17 | * Conduct or speech which might be considered sexist, racist, homophobic, transphobic, ableist or otherwise discriminatory or offensive in nature. 18 | * Do not use unwelcome, suggestive, derogatory or inappropriate nicknames or terms. 19 | * Do not show disrespect towards others. (Jokes, innuendo, dismissive attitudes.) 20 | * Intimidation or harassment (online or in-person). 21 | * Disrespect towards differences of opinion. 22 | * Inappropriate attention or contact. Be aware of how your actions affect others. If it makes someone uncomfortable, stop. 23 | * Unsolicited advertisement. 24 | * Do not promote competing services or companies in your messages. 25 | * Do not include names or logos of competing services or companies in user names or pictures. 26 | * Not understanding the differences between constructive criticism and disparagement. 27 | * Sustained disruptions. 28 | * Violence, threats of violence or violent language. 29 | 30 | ## Enforcement 31 | * Understand that speech and actions have consequences, and unacceptable behavior will not be tolerated. 32 | * If you are the subject of, or witness to any violations of this Code of Conduct, please contact us via email at [codeofconduct@clickhouse.com](mailto:codeofconduct@clickhouse.com). 33 | * The details of any report will be kept in strict confidence unless required by regional law to be reported to appropriate authorities. 34 | * If violations occur, organizers will take any action they deem appropriate for the infraction, up to and including expulsion from channels or events. 35 | 36 | Portions derived from the [Slack Code of Conduct](https://api.slack.com/community/code-of-conduct), [Django Code of Conduct](https://www.djangoproject.com/conduct/), [The Rust Code of Conduct](https://www.rust-lang.org/conduct.html) and [The Ada Initiative](http://adainitiative.org/2014/02/18/howto-design-a-code-of-conduct-for-your-community/) under a Creative Commons [Attribution-ShareAlike](http://creativecommons.org/licenses/by-sa/3.0/) license. 37 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to ClickHouse Copier 2 | 3 | You can send a pull request or report an issue at your own risk. 4 | 5 | While the tool is no longer supported, you should not expect any help. 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016-2024 ClickHouse, Inc. 2 | 3 | Apache License 4 | Version 2.0, January 2004 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, 12 | and distribution as defined by Sections 1 through 9 of this document. 13 | 14 | "Licensor" shall mean the copyright owner or entity authorized by 15 | the copyright owner that is granting the License. 16 | 17 | "Legal Entity" shall mean the union of the acting entity and all 18 | other entities that control, are controlled by, or are under common 19 | control with that entity. For the purposes of this definition, 20 | "control" means (i) the power, direct or indirect, to cause the 21 | direction or management of such entity, whether by contract or 22 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 23 | outstanding shares, or (iii) beneficial ownership of such entity. 24 | 25 | "You" (or "Your") shall mean an individual or Legal Entity 26 | exercising permissions granted by this License. 27 | 28 | "Source" form shall mean the preferred form for making modifications, 29 | including but not limited to software source code, documentation 30 | source, and configuration files. 31 | 32 | "Object" form shall mean any form resulting from mechanical 33 | transformation or translation of a Source form, including but 34 | not limited to compiled object code, generated documentation, 35 | and conversions to other media types. 36 | 37 | "Work" shall mean the work of authorship, whether in Source or 38 | Object form, made available under the License, as indicated by a 39 | copyright notice that is included in or attached to the work 40 | (an example is provided in the Appendix below). 41 | 42 | "Derivative Works" shall mean any work, whether in Source or Object 43 | form, that is based on (or derived from) the Work and for which the 44 | editorial revisions, annotations, elaborations, or other modifications 45 | represent, as a whole, an original work of authorship. For the purposes 46 | of this License, Derivative Works shall not include works that remain 47 | separable from, or merely link (or bind by name) to the interfaces of, 48 | the Work and Derivative Works thereof. 49 | 50 | "Contribution" shall mean any work of authorship, including 51 | the original version of the Work and any modifications or additions 52 | to that Work or Derivative Works thereof, that is intentionally 53 | submitted to Licensor for inclusion in the Work by the copyright owner 54 | or by an individual or Legal Entity authorized to submit on behalf of 55 | the copyright owner. For the purposes of this definition, "submitted" 56 | means any form of electronic, verbal, or written communication sent 57 | to the Licensor or its representatives, including but not limited to 58 | communication on electronic mailing lists, source code control systems, 59 | and issue tracking systems that are managed by, or on behalf of, the 60 | Licensor for the purpose of discussing and improving the Work, but 61 | excluding communication that is conspicuously marked or otherwise 62 | designated in writing by the copyright owner as "Not a Contribution." 63 | 64 | "Contributor" shall mean Licensor and any individual or Legal Entity 65 | on behalf of whom a Contribution has been received by Licensor and 66 | subsequently incorporated within the Work. 67 | 68 | 2. Grant of Copyright License. Subject to the terms and conditions of 69 | this License, each Contributor hereby grants to You a perpetual, 70 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 71 | copyright license to reproduce, prepare Derivative Works of, 72 | publicly display, publicly perform, sublicense, and distribute the 73 | Work and such Derivative Works in Source or Object form. 74 | 75 | 3. Grant of Patent License. Subject to the terms and conditions of 76 | this License, each Contributor hereby grants to You a perpetual, 77 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 78 | (except as stated in this section) patent license to make, have made, 79 | use, offer to sell, sell, import, and otherwise transfer the Work, 80 | where such license applies only to those patent claims licensable 81 | by such Contributor that are necessarily infringed by their 82 | Contribution(s) alone or by combination of their Contribution(s) 83 | with the Work to which such Contribution(s) was submitted. If You 84 | institute patent litigation against any entity (including a 85 | cross-claim or counterclaim in a lawsuit) alleging that the Work 86 | or a Contribution incorporated within the Work constitutes direct 87 | or contributory patent infringement, then any patent licenses 88 | granted to You under this License for that Work shall terminate 89 | as of the date such litigation is filed. 90 | 91 | 4. Redistribution. You may reproduce and distribute copies of the 92 | Work or Derivative Works thereof in any medium, with or without 93 | modifications, and in Source or Object form, provided that You 94 | meet the following conditions: 95 | 96 | (a) You must give any other recipients of the Work or 97 | Derivative Works a copy of this License; and 98 | 99 | (b) You must cause any modified files to carry prominent notices 100 | stating that You changed the files; and 101 | 102 | (c) You must retain, in the Source form of any Derivative Works 103 | that You distribute, all copyright, patent, trademark, and 104 | attribution notices from the Source form of the Work, 105 | excluding those notices that do not pertain to any part of 106 | the Derivative Works; and 107 | 108 | (d) If the Work includes a "NOTICE" text file as part of its 109 | distribution, then any Derivative Works that You distribute must 110 | include a readable copy of the attribution notices contained 111 | within such NOTICE file, excluding those notices that do not 112 | pertain to any part of the Derivative Works, in at least one 113 | of the following places: within a NOTICE text file distributed 114 | as part of the Derivative Works; within the Source form or 115 | documentation, if provided along with the Derivative Works; or, 116 | within a display generated by the Derivative Works, if and 117 | wherever such third-party notices normally appear. The contents 118 | of the NOTICE file are for informational purposes only and 119 | do not modify the License. You may add Your own attribution 120 | notices within Derivative Works that You distribute, alongside 121 | or as an addendum to the NOTICE text from the Work, provided 122 | that such additional attribution notices cannot be construed 123 | as modifying the License. 124 | 125 | You may add Your own copyright statement to Your modifications and 126 | may provide additional or different license terms and conditions 127 | for use, reproduction, or distribution of Your modifications, or 128 | for any such Derivative Works as a whole, provided Your use, 129 | reproduction, and distribution of the Work otherwise complies with 130 | the conditions stated in this License. 131 | 132 | 5. Submission of Contributions. Unless You explicitly state otherwise, 133 | any Contribution intentionally submitted for inclusion in the Work 134 | by You to the Licensor shall be under the terms and conditions of 135 | this License, without any additional terms or conditions. 136 | Notwithstanding the above, nothing herein shall supersede or modify 137 | the terms of any separate license agreement you may have executed 138 | with Licensor regarding such Contributions. 139 | 140 | 6. Trademarks. This License does not grant permission to use the trade 141 | names, trademarks, service marks, or product names of the Licensor, 142 | except as required for reasonable and customary use in describing the 143 | origin of the Work and reproducing the content of the NOTICE file. 144 | 145 | 7. Disclaimer of Warranty. Unless required by applicable law or 146 | agreed to in writing, Licensor provides the Work (and each 147 | Contributor provides its Contributions) on an "AS IS" BASIS, 148 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 149 | implied, including, without limitation, any warranties or conditions 150 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 151 | PARTICULAR PURPOSE. You are solely responsible for determining the 152 | appropriateness of using or redistributing the Work and assume any 153 | risks associated with Your exercise of permissions under this License. 154 | 155 | 8. Limitation of Liability. In no event and under no legal theory, 156 | whether in tort (including negligence), contract, or otherwise, 157 | unless required by applicable law (such as deliberate and grossly 158 | negligent acts) or agreed to in writing, shall any Contributor be 159 | liable to You for damages, including any direct, indirect, special, 160 | incidental, or consequential damages of any character arising as a 161 | result of this License or out of the use or inability to use the 162 | Work (including but not limited to damages for loss of goodwill, 163 | work stoppage, computer failure or malfunction, or any and all 164 | other commercial damages or losses), even if such Contributor 165 | has been advised of the possibility of such damages. 166 | 167 | 9. Accepting Warranty or Additional Liability. While redistributing 168 | the Work or Derivative Works thereof, You may choose to offer, 169 | and charge a fee for, acceptance of support, warranty, indemnity, 170 | or other liability obligations and/or rights consistent with this 171 | License. However, in accepting such obligations, You may act only 172 | on Your own behalf and on Your sole responsibility, not on behalf 173 | of any other Contributor, and only if You agree to indemnify, 174 | defend, and hold each Contributor harmless for any liability 175 | incurred by, or claims asserted against, such Contributor by reason 176 | of your accepting any such warranty or additional liability. 177 | 178 | END OF TERMS AND CONDITIONS 179 | 180 | APPENDIX: How to apply the Apache License to your work. 181 | 182 | To apply the Apache License to your work, attach the following 183 | boilerplate notice, with the fields enclosed by brackets "[]" 184 | replaced with your own identifying information. (Don't include 185 | the brackets!) The text should be enclosed in the appropriate 186 | comment syntax for the file format. We also recommend that a 187 | file or class name and description of purpose be included on the 188 | same "printed page" as the copyright notice for easier 189 | identification within third-party archives. 190 | 191 | Copyright 2016-2024 ClickHouse, Inc. 192 | 193 | Licensed under the Apache License, Version 2.0 (the "License"); 194 | you may not use this file except in compliance with the License. 195 | You may obtain a copy of the License at 196 | 197 | http://www.apache.org/licenses/LICENSE-2.0 198 | 199 | Unless required by applicable law or agreed to in writing, software 200 | distributed under the License is distributed on an "AS IS" BASIS, 201 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 202 | See the License for the specific language governing permissions and 203 | limitations under the License. 204 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > [!NOTE] 2 | > This tool is no longer supported, but you can use the latest available version as is. 3 | 4 | # clickhouse-copier 5 | 6 | Copies data from the tables in one cluster to tables in another (or the same) cluster. 7 | 8 | To get a consistent copy, the data in the source tables and partitions should not change during the entire process. 9 | 10 | You can run multiple `clickhouse-copier` instances on different servers to perform the same job. ClickHouse Keeper, or ZooKeeper, is used for syncing the processes. 11 | 12 | After starting, `clickhouse-copier`: 13 | 14 | - Connects to ClickHouse Keeper and receives: 15 | 16 | - Copying jobs. 17 | - The state of the copying jobs. 18 | 19 | - It performs the jobs. 20 | 21 | Each running process chooses the “closest” shard of the source cluster and copies the data into the destination cluster, resharding the data if necessary. 22 | 23 | `clickhouse-copier` tracks the changes in ClickHouse Keeper and applies them on the fly. 24 | 25 | To reduce network traffic, we recommend running `clickhouse-copier` on the same server where the source data is located. 26 | 27 | ## Download and Install 28 | 29 | Download the binaries from the [final release](releases/tag/final). 30 | 31 | ## Running Clickhouse-copier 32 | 33 | The utility should be run manually: 34 | 35 | ``` bash 36 | $ clickhouse-copier --daemon --config keeper.xml --task-path /task/path --base-dir /path/to/dir 37 | ``` 38 | 39 | Parameters: 40 | 41 | - `daemon` — Starts `clickhouse-copier` in daemon mode. 42 | - `config` — The path to the `keeper.xml` file with the parameters for the connection to ClickHouse Keeper. 43 | - `task-path` — The path to the ClickHouse Keeper node. This node is used for syncing `clickhouse-copier` processes and storing tasks. Tasks are stored in `$task-path/description`. 44 | - `task-file` — Optional path to file with task configuration for initial upload to ClickHouse Keeper. 45 | - `task-upload-force` — Force upload `task-file` even if node already exists. Default is false. 46 | - `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` creates `clickhouse-copier_YYYYMMHHSS_` subdirectories in `$base-dir`. If this parameter is omitted, the directories are created in the directory where `clickhouse-copier` was launched. 47 | 48 | ## Format of keeper.xml 49 | 50 | ``` xml 51 | 52 | 53 | trace 54 | 100M 55 | 3 56 | 57 | 58 | 59 | 60 | 127.0.0.1 61 | 2181 62 | 63 | 64 | 65 | ``` 66 | 67 | ## Configuration of Copying Tasks 68 | 69 | ``` xml 70 | 71 | 72 | 73 | 74 | 79 | 80 | false 81 | 82 | 127.0.0.1 83 | 9000 84 | 89 | 90 | 91 | ... 92 | 93 | 94 | 95 | ... 96 | 97 | 98 | 99 | 100 | 2 101 | 102 | 103 | 104 | 1 105 | 106 | 107 | 108 | 109 | 0 110 | 111 | 112 | 114 | 115 | 3 116 | 117 | 1 118 | 119 | 120 | 124 | 125 | 126 | 127 | 128 | source_cluster 129 | test 130 | hits 131 | 132 | 133 | destination_cluster 134 | test 135 | hits2 136 | 137 | 146 | 147 | ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}') 148 | PARTITION BY toMonday(date) 149 | ORDER BY (CounterID, EventDate) 150 | 151 | 152 | 153 | jumpConsistentHash(intHash64(UserID), 2) 154 | 155 | 156 | CounterID != 0 157 | 158 | 170 | 171 | '2018-02-26' 172 | '2018-03-05' 173 | ... 174 | 175 | 176 | 177 | 178 | 179 | ... 180 | 181 | ... 182 | 183 | 184 | ``` 185 | 186 | `clickhouse-copier` tracks the changes in `/task/path/description` and applies them on the fly. For instance, if you change the value of `max_workers`, the number of processes running tasks will also change. 187 | 188 | ## Build from sources 189 | 190 | You don't have to. Download the binaries from the [final release](releases/tag/final). 191 | 192 | But if you want, use the following repository snapshot https://github.com/ClickHouse/ClickHouse/tree/1179a70c21eeca88410a012a73a49180cc5e5e2e and proceed with the normal ClickHouse build. The built `clickhouse` binary will contain the copier tool. 193 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Security Announcements 4 | 5 | There are none. 6 | 7 | ## Scope and Supported Versions 8 | 9 | This tool is no longer supported. 10 | 11 | ## Reporting a Vulnerability 12 | 13 | It does not make sense to report any findings. 14 | 15 | ### When Should I Report a Vulnerability? 16 | 17 | You should not. 18 | 19 | ### When Should I NOT Report a Vulnerability? 20 | 21 | Yes. 22 | 23 | ## Security Vulnerability Response 24 | 25 | No responses will be provided. 26 | 27 | ## Public Disclosure Timing 28 | 29 | No timings or obligations whatsoever. 30 | -------------------------------------------------------------------------------- /docs/en/operations/utilities/clickhouse-copier.md: -------------------------------------------------------------------------------- 1 | ../../../../README.md -------------------------------------------------------------------------------- /docs/ru/operations/utilities/clickhouse-copier.md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: /ru/operations/utilities/clickhouse-copier 3 | sidebar_position: 59 4 | sidebar_label: clickhouse-copier 5 | --- 6 | 7 | # clickhouse-copier {#clickhouse-copier} 8 | 9 | Копирует данные из таблиц одного кластера в таблицы другого (или этого же) кластера. 10 | 11 | Можно запустить несколько `clickhouse-copier` для разных серверах для выполнения одного и того же задания. Для синхронизации между процессами используется ZooKeeper. 12 | 13 | После запуска, `clickhouse-copier`: 14 | 15 | - Соединяется с ZooKeeper и получает: 16 | 17 | - Задания на копирование. 18 | - Состояние заданий на копирование. 19 | 20 | - Выполняет задания. 21 | 22 | Каждый запущенный процесс выбирает "ближайший" шард исходного кластера и копирует данные в кластер назначения, при необходимости перешардируя их. 23 | 24 | `clickhouse-copier` отслеживает изменения в ZooKeeper и применяет их «на лету». 25 | 26 | Для снижения сетевого трафика рекомендуем запускать `clickhouse-copier` на том же сервере, где находятся исходные данные. 27 | 28 | ## Запуск Clickhouse-copier {#zapusk-clickhouse-copier} 29 | 30 | Утилиту следует запускать вручную следующим образом: 31 | 32 | ``` bash 33 | $ clickhouse-copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir 34 | ``` 35 | 36 | Параметры запуска: 37 | 38 | - `daemon` - запускает `clickhouse-copier` в режиме демона. 39 | - `config` - путь к файлу `zookeeper.xml` с параметрами соединения с ZooKeeper. 40 | - `task-path` - путь к ноде ZooKeeper. Нода используется для синхронизации между процессами `clickhouse-copier` и для хранения заданий. Задания хранятся в `$task-path/description`. 41 | - `task-file` - необязательный путь к файлу с описанием конфигурация заданий для загрузки в ZooKeeper. 42 | - `task-upload-force` - Загрузить `task-file` в ZooKeeper даже если уже было загружено. 43 | - `base-dir` - путь к логам и вспомогательным файлам. При запуске `clickhouse-copier` создает в `$base-dir` подкаталоги `clickhouse-copier_YYYYMMHHSS_`. Если параметр не указан, то каталоги будут создаваться в каталоге, где `clickhouse-copier` был запущен. 44 | 45 | ## Формат Zookeeper.xml {#format-zookeeper-xml} 46 | 47 | ``` xml 48 | 49 | 50 | trace 51 | 100M 52 | 3 53 | 54 | 55 | 56 | 57 | 127.0.0.1 58 | 2181 59 | 60 | 61 | 62 | ``` 63 | 64 | ## Конфигурация заданий на копирование {#konfiguratsiia-zadanii-na-kopirovanie} 65 | 66 | ``` xml 67 | 68 | 69 | 70 | 71 | 76 | 77 | false 78 | 79 | 127.0.0.1 80 | 9000 81 | 86 | 87 | 88 | ... 89 | 90 | 91 | 92 | ... 93 | 94 | 95 | 96 | 97 | 2 98 | 99 | 100 | 101 | 1 102 | 103 | 104 | 105 | 106 | 0 107 | 108 | 109 | 111 | 112 | 3 113 | 114 | 1 115 | 116 | 117 | 121 | 122 | 123 | 124 | 125 | source_cluster 126 | test 127 | hits 128 | 129 | 130 | destination_cluster 131 | test 132 | hits2 133 | 134 | 143 | 144 | ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}') 145 | PARTITION BY toMonday(date) 146 | ORDER BY (CounterID, EventDate) 147 | 148 | 149 | 150 | jumpConsistentHash(intHash64(UserID), 2) 151 | 152 | 153 | CounterID != 0 154 | 155 | 167 | 168 | '2018-02-26' 169 | '2018-03-05' 170 | ... 171 | 172 | 173 | 174 | 175 | 176 | ... 177 | 178 | ... 179 | 180 | 181 | ``` 182 | 183 | `clickhouse-copier` отслеживает изменения `/task/path/description` и применяет их «на лету». Если вы поменяете, например, значение `max_workers`, то количество процессов, выполняющих задания, также изменится. 184 | -------------------------------------------------------------------------------- /docs/zh/operations/utilities/clickhouse-copier.md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: /zh/operations/utilities/clickhouse-copier 3 | --- 4 | # clickhouse-copier {#clickhouse-copier} 5 | 6 | 将数据从一个群集中的表复制到另一个（或相同）群集中的表。 7 | 8 | 您可以运行多个 `clickhouse-copier` 不同服务器上的实例执行相同的作业。 ZooKeeper用于同步进程。 9 | 10 | 开始后, `clickhouse-copier`: 11 | 12 | - 连接到ZooKeeper并且接收: 13 | 14 | - 复制作业。 15 | - 复制作业的状态。 16 | 17 | - 它执行的工作。 18 | 19 | 每个正在运行的进程都会选择源集群的“最接近”分片，然后将数据复制到目标集群，并在必要时重新分片数据。 20 | 21 | `clickhouse-copier` 跟踪ZooKeeper中的更改，并实时应用它们。 22 | 23 | 为了减少网络流量，我们建议运行 `clickhouse-copier` 在源数据所在的同一服务器上。 24 | 25 | ## 运行Clickhouse-copier {#running-clickhouse-copier} 26 | 27 | 该实用程序应手动运行: 28 | 29 | ``` bash 30 | clickhouse-copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir 31 | ``` 32 | 33 | 参数: 34 | 35 | - `daemon` — 在守护进程模式下启动`clickhouse-copier`。 36 | - `config` — `zookeeper.xml`文件的路径，其中包含用于连接ZooKeeper的参数。 37 | - `task-path` — ZooKeeper节点的路径。 该节点用于同步`clickhouse-copier`进程和存储任务。 任务存储在`$task-path/description`中。 38 | - `task-file` — 可选的非必须参数, 指定一个包含任务配置的参数文件, 用于初始上传到ZooKeeper。 39 | - `task-upload-force` — 即使节点已经存在，也强制上载`task-file`。 40 | - `base-dir` — 日志和辅助文件的路径。 启动时，`clickhouse-copier`在`$base-dir`中创建`clickhouse-copier_YYYYMMHHSS_`子目录。 如果省略此参数，则会在启动`clickhouse-copier`的目录中创建目录。 41 | 42 | 43 | 44 | ## Zookeeper.xml格式 {#format-of-zookeeper-xml} 45 | 46 | ``` xml 47 | 48 | 49 | trace 50 | 100M 51 | 3 52 | 53 | 54 | 55 | 56 | 127.0.0.1 57 | 2181 58 | 59 | 60 | 61 | ``` 62 | 63 | ## 复制任务的配置 {#configuration-of-copying-tasks} 64 | 65 | ``` xml 66 | 67 | 68 | 69 | 70 | 71 | false 72 | 73 | 127.0.0.1 74 | 9000 75 | 76 | 77 | ... 78 | 79 | 80 | 81 | ... 82 | 83 | 84 | 85 | 86 | 2 87 | 88 | 89 | 90 | 1 91 | 92 | 93 | 94 | 95 | 0 96 | 97 | 98 | 100 | 101 | 3 102 | 103 | 1 104 | 105 | 106 | 110 | 111 | 112 | 113 | 114 | source_cluster 115 | test 116 | hits 117 | 118 | 119 | destination_cluster 120 | test 121 | hits2 122 | 123 | 132 | 133 | ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}') 134 | PARTITION BY toMonday(date) 135 | ORDER BY (CounterID, EventDate) 136 | 137 | 138 | 139 | jumpConsistentHash(intHash64(UserID), 2) 140 | 141 | 142 | CounterID != 0 143 | 144 | 156 | 157 | '2018-02-26' 158 | '2018-03-05' 159 | ... 160 | 161 | 162 | 163 | 164 | 165 | ... 166 | 167 | ... 168 | 169 | 170 | ``` 171 | 172 | `clickhouse-copier` 跟踪更改 `/task/path/description` 并在飞行中应用它们。 例如，如果你改变的值 `max_workers`，运行任务的进程数也会发生变化。 173 | -------------------------------------------------------------------------------- /programs/copier/Aliases.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | 7 | #include 8 | 9 | namespace DB 10 | { 11 | using ConfigurationPtr = Poco::AutoPtr; 12 | 13 | using DatabaseAndTableName = std::pair; 14 | using ListOfDatabasesAndTableNames = std::vector; 15 | } 16 | -------------------------------------------------------------------------------- /programs/copier/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(CLICKHOUSE_COPIER_SOURCES 2 | "${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopierApp.cpp" 3 | "${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopier.cpp" 4 | "${CMAKE_CURRENT_SOURCE_DIR}/Internals.cpp" 5 | "${CMAKE_CURRENT_SOURCE_DIR}/ShardPartition.cpp" 6 | "${CMAKE_CURRENT_SOURCE_DIR}/ShardPartitionPiece.cpp" 7 | "${CMAKE_CURRENT_SOURCE_DIR}/StatusAccumulator.cpp" 8 | "${CMAKE_CURRENT_SOURCE_DIR}/TaskCluster.cpp" 9 | "${CMAKE_CURRENT_SOURCE_DIR}/TaskShard.cpp" 10 | "${CMAKE_CURRENT_SOURCE_DIR}/TaskTable.cpp") 11 | 12 | set (CLICKHOUSE_COPIER_LINK 13 | PRIVATE 14 | clickhouse_common_zookeeper 15 | clickhouse_common_config 16 | clickhouse_parsers 17 | clickhouse_functions 18 | clickhouse_table_functions 19 | clickhouse_aggregate_functions 20 | string_utils 21 | 22 | PUBLIC 23 | daemon 24 | ) 25 | 26 | set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) 27 | 28 | clickhouse_program_add(copier) 29 | -------------------------------------------------------------------------------- /programs/copier/ClusterCopier.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Aliases.h" 4 | #include "Internals.h" 5 | #include "TaskCluster.h" 6 | #include "TaskShard.h" 7 | #include "TaskTable.h" 8 | #include "ShardPartition.h" 9 | #include "ShardPartitionPiece.h" 10 | #include "ZooKeeperStaff.h" 11 | 12 | 13 | namespace DB 14 | { 15 | 16 | class ClusterCopier : WithMutableContext 17 | { 18 | public: 19 | ClusterCopier(const String & task_path_, 20 | const String & host_id_, 21 | const String & proxy_database_name_, 22 | ContextMutablePtr context_, 23 | LoggerRawPtr log_) 24 | : WithMutableContext(context_), 25 | task_zookeeper_path(task_path_), 26 | host_id(host_id_), 27 | working_database_name(proxy_database_name_), 28 | log(log_) {} 29 | 30 | void init(); 31 | 32 | template 33 | decltype(auto) retry(T && func, UInt64 max_tries = 100); 34 | 35 | void discoverShardPartitions(const ConnectionTimeouts & timeouts, const TaskShardPtr & task_shard); 36 | 37 | /// Compute set of partitions, assume set of partitions aren't changed during the processing 38 | void discoverTablePartitions(const ConnectionTimeouts & timeouts, TaskTable & task_table, UInt64 num_threads = 0); 39 | 40 | void uploadTaskDescription(const std::string & task_path, const std::string & task_file, bool force); 41 | 42 | void reloadTaskDescription(); 43 | 44 | void updateConfigIfNeeded(); 45 | 46 | void process(const ConnectionTimeouts & timeouts); 47 | 48 | /// Disables DROP PARTITION commands that used to clear data after errors 49 | void setSafeMode(bool is_safe_mode_ = true) 50 | { 51 | is_safe_mode = is_safe_mode_; 52 | } 53 | 54 | void setCopyFaultProbability(double copy_fault_probability_) 55 | { 56 | copy_fault_probability = copy_fault_probability_; 57 | } 58 | 59 | void setMoveFaultProbability(double move_fault_probability_) 60 | { 61 | move_fault_probability = move_fault_probability_; 62 | } 63 | 64 | void setExperimentalUseSampleOffset(bool value) 65 | { 66 | experimental_use_sample_offset = value; 67 | } 68 | 69 | void setMaxTableTries(UInt64 tries) 70 | { 71 | max_table_tries = tries; 72 | } 73 | void setMaxShardPartitionTries(UInt64 tries) 74 | { 75 | max_shard_partition_tries = tries; 76 | } 77 | void setMaxShardPartitionPieceTriesForAlter(UInt64 tries) 78 | { 79 | max_shard_partition_piece_tries_for_alter = tries; 80 | } 81 | void setRetryDelayMs(std::chrono::milliseconds ms) 82 | { 83 | retry_delay_ms = ms; 84 | } 85 | 86 | protected: 87 | 88 | String getWorkersPath() const 89 | { 90 | return task_cluster->task_zookeeper_path + "/task_active_workers"; 91 | } 92 | 93 | String getWorkersPathVersion() const 94 | { 95 | return getWorkersPath() + "_version"; 96 | } 97 | 98 | String getCurrentWorkerNodePath() const 99 | { 100 | return getWorkersPath() + "/" + host_id; 101 | } 102 | 103 | zkutil::EphemeralNodeHolder::Ptr createTaskWorkerNodeAndWaitIfNeed( 104 | const zkutil::ZooKeeperPtr & zookeeper, 105 | const String & description, 106 | bool unprioritized); 107 | 108 | /* 109 | * Checks that partition piece or some other entity is clean. 110 | * The only requirement is that you have to pass is_dirty_flag_path and is_dirty_cleaned_path to the function. 111 | * And is_dirty_flag_path is a parent of is_dirty_cleaned_path. 112 | * */ 113 | static bool checkPartitionPieceIsClean( 114 | const zkutil::ZooKeeperPtr & zookeeper, 115 | const CleanStateClock & clean_state_clock, 116 | const String & task_status_path); 117 | 118 | bool checkAllPiecesInPartitionAreDone(const TaskTable & task_table, const String & partition_name, const TasksShard & shards_with_partition); 119 | 120 | /** Checks that the whole partition of a table was copied. We should do it carefully due to dirty lock. 121 | * State of some task could change during the processing. 122 | * We have to ensure that all shards have the finished state and there is no dirty flag. 123 | * Moreover, we have to check status twice and check zxid, because state can change during the checking. 124 | */ 125 | 126 | /* The same as function above 127 | * Assume that we don't know on which shards do we have partition certain piece. 128 | * We'll check them all (I mean shards that contain the whole partition) 129 | * And shards that don't have certain piece MUST mark that piece is_done true. 130 | * */ 131 | bool checkPartitionPieceIsDone(const TaskTable & task_table, const String & partition_name, 132 | size_t piece_number, const TasksShard & shards_with_partition); 133 | 134 | 135 | /*Alter successful insertion to helping tables it will move all pieces to destination table*/ 136 | TaskStatus tryMoveAllPiecesToDestinationTable(const TaskTable & task_table, const String & partition_name); 137 | 138 | /// Removes MATERIALIZED and ALIAS columns from create table query 139 | static ASTPtr removeAliasMaterializedAndTTLColumnsFromCreateQuery(const ASTPtr & query_ast, bool allow_to_copy_alias_and_materialized_columns); 140 | 141 | bool tryDropPartitionPiece(ShardPartition & task_partition, size_t current_piece_number, 142 | const zkutil::ZooKeeperPtr & zookeeper, const CleanStateClock & clean_state_clock); 143 | 144 | bool tryProcessTable(const ConnectionTimeouts & timeouts, TaskTable & task_table); 145 | 146 | TaskStatus tryCreateDestinationTable(const ConnectionTimeouts & timeouts, TaskTable & task_table); 147 | /// Job for copying partition from particular shard. 148 | TaskStatus tryProcessPartitionTask(const ConnectionTimeouts & timeouts, 149 | ShardPartition & task_partition, 150 | bool is_unprioritized_task); 151 | 152 | TaskStatus iterateThroughAllPiecesInPartition(const ConnectionTimeouts & timeouts, 153 | ShardPartition & task_partition, 154 | bool is_unprioritized_task); 155 | 156 | TaskStatus processPartitionPieceTaskImpl(const ConnectionTimeouts & timeouts, 157 | ShardPartition & task_partition, 158 | size_t current_piece_number, 159 | bool is_unprioritized_task); 160 | 161 | void dropAndCreateLocalTable(const ASTPtr & create_ast); 162 | 163 | void dropLocalTableIfExists(const DatabaseAndTableName & table_name) const; 164 | 165 | void dropHelpingTables(const TaskTable & task_table); 166 | 167 | void dropHelpingTablesByPieceNumber(const TaskTable & task_table, size_t current_piece_number); 168 | 169 | /// Is used for usage less disk space. 170 | /// After all pieces were successfully moved to original destination 171 | /// table we can get rid of partition pieces (partitions in helping tables). 172 | void dropParticularPartitionPieceFromAllHelpingTables(const TaskTable & task_table, const String & partition_name); 173 | 174 | String getRemoteCreateTable(const DatabaseAndTableName & table, Connection & connection, const Settings & settings); 175 | 176 | ASTPtr getCreateTableForPullShard(const ConnectionTimeouts & timeouts, TaskShard & task_shard); 177 | 178 | /// If it is implicitly asked to create split Distributed table for certain piece on current shard, we will do it. 179 | void createShardInternalTables(const ConnectionTimeouts & timeouts, TaskShard & task_shard, bool create_split = true); 180 | 181 | std::set getShardPartitions(const ConnectionTimeouts & timeouts, TaskShard & task_shard); 182 | 183 | bool checkShardHasPartition(const ConnectionTimeouts & timeouts, TaskShard & task_shard, const String & partition_quoted_name); 184 | 185 | bool checkPresentPartitionPiecesOnCurrentShard(const ConnectionTimeouts & timeouts, 186 | TaskShard & task_shard, const String & partition_quoted_name, size_t current_piece_number); 187 | 188 | /* 189 | * This class is used in executeQueryOnCluster function 190 | * You can execute query on each shard (no sense it is executed on each replica of a shard or not) 191 | * or you can execute query on each replica on each shard. 192 | * First mode is useful for INSERTS queries. 193 | * */ 194 | enum ClusterExecutionMode 195 | { 196 | ON_EACH_SHARD, 197 | ON_EACH_NODE 198 | }; 199 | 200 | /** Executes simple query (without output streams, for example DDL queries) on each shard of the cluster 201 | * Returns number of shards for which at least one replica executed query successfully 202 | */ 203 | UInt64 executeQueryOnCluster( 204 | const ClusterPtr & cluster, 205 | const String & query, 206 | const Settings & current_settings, 207 | ClusterExecutionMode execution_mode = ClusterExecutionMode::ON_EACH_SHARD) const; 208 | 209 | private: 210 | String task_zookeeper_path; 211 | String task_description_path; 212 | String host_id; 213 | String working_database_name; 214 | 215 | /// Auto update config stuff 216 | UInt64 task_description_current_version = 1; 217 | std::atomic task_description_version{1}; 218 | Coordination::WatchCallback task_description_watch_callback; 219 | /// ZooKeeper session used to set the callback 220 | zkutil::ZooKeeperPtr task_description_watch_zookeeper; 221 | 222 | ConfigurationPtr task_cluster_initial_config; 223 | ConfigurationPtr task_cluster_current_config; 224 | 225 | std::unique_ptr task_cluster; 226 | 227 | bool is_safe_mode = false; 228 | double copy_fault_probability = 0.0; 229 | double move_fault_probability = 0.0; 230 | 231 | bool experimental_use_sample_offset{false}; 232 | 233 | LoggerRawPtr log; 234 | 235 | UInt64 max_table_tries = 3; 236 | UInt64 max_shard_partition_tries = 3; 237 | UInt64 max_shard_partition_piece_tries_for_alter = 10; 238 | std::chrono::milliseconds retry_delay_ms{1000}; 239 | }; 240 | } 241 | -------------------------------------------------------------------------------- /programs/copier/ClusterCopierApp.cpp: -------------------------------------------------------------------------------- 1 | #include "ClusterCopierApp.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace fs = std::filesystem; 14 | 15 | namespace DB 16 | { 17 | 18 | /// ClusterCopierApp 19 | 20 | void ClusterCopierApp::initialize(Poco::Util::Application & self) 21 | { 22 | is_help = config().has("help"); 23 | if (is_help) 24 | return; 25 | 26 | config_xml_path = config().getString("config-file"); 27 | task_path = config().getString("task-path"); 28 | log_level = config().getString("log-level", "info"); 29 | is_safe_mode = config().has("safe-mode"); 30 | is_status_mode = config().has("status"); 31 | if (config().has("copy-fault-probability")) 32 | copy_fault_probability = std::max(std::min(config().getDouble("copy-fault-probability"), 1.0), 0.0); 33 | if (config().has("move-fault-probability")) 34 | move_fault_probability = std::max(std::min(config().getDouble("move-fault-probability"), 1.0), 0.0); 35 | base_dir = (config().has("base-dir")) ? config().getString("base-dir") : fs::current_path().string(); 36 | 37 | max_table_tries = std::max(config().getUInt("max-table-tries", 3), 1); 38 | max_shard_partition_tries = std::max(config().getUInt("max-shard-partition-tries", 3), 1); 39 | max_shard_partition_piece_tries_for_alter = std::max(config().getUInt("max-shard-partition-piece-tries-for-alter", 10), 1); 40 | retry_delay_ms = std::chrono::milliseconds(std::max(config().getUInt("retry-delay-ms", 1000), 100)); 41 | 42 | if (config().has("experimental-use-sample-offset")) 43 | experimental_use_sample_offset = config().getBool("experimental-use-sample-offset"); 44 | 45 | // process_id is '#_' 46 | time_t timestamp = Poco::Timestamp().epochTime(); 47 | auto curr_pid = Poco::Process::id(); 48 | 49 | process_id = std::to_string(DateLUT::serverTimezoneInstance().toNumYYYYMMDDhhmmss(timestamp)) + "_" + std::to_string(curr_pid); 50 | host_id = escapeForFileName(getFQDNOrHostName()) + '#' + process_id; 51 | process_path = fs::weakly_canonical(fs::path(base_dir) / ("clickhouse-copier_" + process_id)); 52 | fs::create_directories(process_path); 53 | 54 | /// Override variables for BaseDaemon 55 | if (config().has("log-level")) 56 | config().setString("logger.level", config().getString("log-level")); 57 | 58 | if (config().has("base-dir") || !config().has("logger.log")) 59 | config().setString("logger.log", fs::path(process_path) / "log.log"); 60 | 61 | if (config().has("base-dir") || !config().has("logger.errorlog")) 62 | config().setString("logger.errorlog", fs::path(process_path) / "log.err.log"); 63 | 64 | Base::initialize(self); 65 | } 66 | 67 | 68 | void ClusterCopierApp::handleHelp(const std::string &, const std::string &) 69 | { 70 | uint16_t terminal_width = 0; 71 | if (isatty(STDIN_FILENO)) 72 | terminal_width = getTerminalWidth(); 73 | 74 | Poco::Util::HelpFormatter help_formatter(options()); 75 | if (terminal_width) 76 | help_formatter.setWidth(terminal_width); 77 | help_formatter.setCommand(commandName()); 78 | help_formatter.setHeader("Copies tables from one cluster to another"); 79 | help_formatter.setUsage("--config-file --task-path "); 80 | help_formatter.format(std::cerr); 81 | help_formatter.setFooter("See also: https://clickhouse.com/docs/en/operations/utilities/clickhouse-copier/"); 82 | 83 | stopOptionsProcessing(); 84 | } 85 | 86 | 87 | void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options) 88 | { 89 | Base::defineOptions(options); 90 | 91 | options.addOption(Poco::Util::Option("task-path", "", "path to task in ZooKeeper") 92 | .argument("task-path").binding("task-path")); 93 | options.addOption(Poco::Util::Option("task-file", "", "path to task file for uploading in ZooKeeper to task-path") 94 | .argument("task-file").binding("task-file")); 95 | options.addOption(Poco::Util::Option("task-upload-force", "", "Force upload task-file even node already exists. Default is false.") 96 | .argument("task-upload-force").binding("task-upload-force")); 97 | options.addOption(Poco::Util::Option("safe-mode", "", "disables ALTER DROP PARTITION in case of errors") 98 | .binding("safe-mode")); 99 | options.addOption(Poco::Util::Option("copy-fault-probability", "", "the copying fails with specified probability (used to test partition state recovering)") 100 | .argument("copy-fault-probability").binding("copy-fault-probability")); 101 | options.addOption(Poco::Util::Option("move-fault-probability", "", "the moving fails with specified probability (used to test partition state recovering)") 102 | .argument("move-fault-probability").binding("move-fault-probability")); 103 | options.addOption(Poco::Util::Option("log-level", "", "sets log level") 104 | .argument("log-level").binding("log-level")); 105 | options.addOption(Poco::Util::Option("base-dir", "", "base directory for copiers, consecutive copier launches will populate /base-dir/launch_id/* directories") 106 | .argument("base-dir").binding("base-dir")); 107 | options.addOption(Poco::Util::Option("experimental-use-sample-offset", "", "Use SAMPLE OFFSET query instead of cityHash64(PRIMARY KEY) % n == k") 108 | .argument("experimental-use-sample-offset").binding("experimental-use-sample-offset")); 109 | options.addOption(Poco::Util::Option("status", "", "Get for status for current execution").binding("status")); 110 | 111 | options.addOption(Poco::Util::Option("max-table-tries", "", "Number of tries for the copy table task") 112 | .argument("max-table-tries").binding("max-table-tries")); 113 | options.addOption(Poco::Util::Option("max-shard-partition-tries", "", "Number of tries for the copy one partition task") 114 | .argument("max-shard-partition-tries").binding("max-shard-partition-tries")); 115 | options.addOption(Poco::Util::Option("max-shard-partition-piece-tries-for-alter", "", "Number of tries for final ALTER ATTACH to destination table") 116 | .argument("max-shard-partition-piece-tries-for-alter").binding("max-shard-partition-piece-tries-for-alter")); 117 | options.addOption(Poco::Util::Option("retry-delay-ms", "", "Delay between task retries") 118 | .argument("retry-delay-ms").binding("retry-delay-ms")); 119 | 120 | using Me = std::decay_t; 121 | options.addOption(Poco::Util::Option("help", "", "produce this help message").binding("help") 122 | .callback(Poco::Util::OptionCallback(this, &Me::handleHelp))); 123 | } 124 | 125 | 126 | void ClusterCopierApp::mainImpl() 127 | { 128 | /// Status command 129 | { 130 | if (is_status_mode) 131 | { 132 | SharedContextHolder shared_context = Context::createShared(); 133 | auto context = Context::createGlobal(shared_context.get()); 134 | context->makeGlobalContext(); 135 | SCOPE_EXIT_SAFE(context->shutdown()); 136 | 137 | auto zookeeper = context->getZooKeeper(); 138 | auto status_json = zookeeper->get(task_path + "/status"); 139 | 140 | LOG_INFO(&logger(), "{}", status_json); 141 | std::cout << status_json << std::endl; 142 | 143 | context->resetZooKeeper(); 144 | return; 145 | } 146 | } 147 | StatusFile status_file(process_path + "/status", StatusFile::write_full_info); 148 | ThreadStatus thread_status; 149 | 150 | auto * log = &logger(); 151 | LOG_INFO(log, "Starting clickhouse-copier (id {}, host_id {}, path {}, revision {})", process_id, host_id, process_path, ClickHouseRevision::getVersionRevision()); 152 | 153 | SharedContextHolder shared_context = Context::createShared(); 154 | auto context = Context::createGlobal(shared_context.get()); 155 | context->makeGlobalContext(); 156 | SCOPE_EXIT_SAFE(context->shutdown()); 157 | 158 | context->setConfig(loaded_config.configuration); 159 | context->setApplicationType(Context::ApplicationType::LOCAL); 160 | context->setPath(process_path + "/"); 161 | 162 | registerInterpreters(); 163 | registerFunctions(); 164 | registerAggregateFunctions(); 165 | registerTableFunctions(); 166 | registerDatabases(); 167 | registerStorages(); 168 | registerDictionaries(); 169 | registerDisks(/* global_skip_access_check= */ true); 170 | registerFormats(); 171 | 172 | static const std::string default_database = "_local"; 173 | DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database, context)); 174 | context->setCurrentDatabase(default_database); 175 | 176 | /// Disable queries logging, since: 177 | /// - There are bits that is not allowed for global context, like adding factories info (for the query_log) 178 | /// - And anyway it is useless for copier. 179 | context->setSetting("log_queries", false); 180 | 181 | auto local_context = Context::createCopy(context); 182 | 183 | /// Initialize query scope just in case. 184 | CurrentThread::QueryScope query_scope(local_context); 185 | 186 | auto copier = std::make_unique( 187 | task_path, host_id, default_database, local_context, log); 188 | copier->setSafeMode(is_safe_mode); 189 | copier->setCopyFaultProbability(copy_fault_probability); 190 | copier->setMoveFaultProbability(move_fault_probability); 191 | copier->setMaxTableTries(max_table_tries); 192 | copier->setMaxShardPartitionTries(max_shard_partition_tries); 193 | copier->setMaxShardPartitionPieceTriesForAlter(max_shard_partition_piece_tries_for_alter); 194 | copier->setRetryDelayMs(retry_delay_ms); 195 | copier->setExperimentalUseSampleOffset(experimental_use_sample_offset); 196 | 197 | auto task_file = config().getString("task-file", ""); 198 | if (!task_file.empty()) 199 | copier->uploadTaskDescription(task_path, task_file, config().getBool("task-upload-force", false)); 200 | 201 | zkutil::validateZooKeeperConfig(config()); 202 | 203 | copier->init(); 204 | copier->process(ConnectionTimeouts::getTCPTimeoutsWithoutFailover(context->getSettingsRef())); 205 | 206 | /// Reset ZooKeeper before removing ClusterCopier. 207 | /// Otherwise zookeeper watch can call callback which use already removed ClusterCopier object. 208 | context->resetZooKeeper(); 209 | } 210 | 211 | 212 | int ClusterCopierApp::main(const std::vector &) 213 | { 214 | if (is_help) 215 | return 0; 216 | 217 | try 218 | { 219 | mainImpl(); 220 | } 221 | catch (...) 222 | { 223 | tryLogCurrentException(&Poco::Logger::root(), __PRETTY_FUNCTION__); 224 | auto code = getCurrentExceptionCode(); 225 | 226 | return (code) ? code : -1; 227 | } 228 | 229 | return 0; 230 | } 231 | 232 | 233 | } 234 | 235 | #pragma GCC diagnostic ignored "-Wunused-function" 236 | #pragma GCC diagnostic ignored "-Wmissing-declarations" 237 | 238 | int mainEntryClickHouseClusterCopier(int argc, char ** argv) 239 | { 240 | try 241 | { 242 | DB::ClusterCopierApp app; 243 | return app.run(argc, argv); 244 | } 245 | catch (...) 246 | { 247 | std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; 248 | auto code = DB::getCurrentExceptionCode(); 249 | 250 | return (code) ? code : -1; 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /programs/copier/ClusterCopierApp.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "ClusterCopier.h" 7 | 8 | /* clickhouse cluster copier util 9 | * Copies tables data from one cluster to new tables of other (possibly the same) cluster in distributed fault-tolerant manner. 10 | * 11 | * See overview in the docs: docs/en/utils/clickhouse-copier.md 12 | * 13 | * Implementation details: 14 | * 15 | * cluster-copier workers pull each partition of each shard of the source cluster and push it to the destination cluster through 16 | * Distributed table (to perform data resharding). So, worker job is a partition of a source shard. 17 | * A job has three states: Active, Finished and Abandoned. Abandoned means that worker died and did not finish the job. 18 | * 19 | * If an error occurred during the copying (a worker failed or a worker did not finish the INSERT), then the whole partition (on 20 | * all destination servers) should be dropped and refilled. So, copying entity is a partition of all destination shards. 21 | * If a failure is detected a special /is_dirty node is created in ZooKeeper signalling that other workers copying the same partition 22 | * should stop, after a refilling procedure should start. 23 | * 24 | * ZooKeeper task node has the following structure: 25 | * /task/path_root - path passed in --task-path parameter 26 | * /description - contains user-defined XML config of the task 27 | * /task_active_workers - contains ephemeral nodes of all currently active workers, used to implement max_workers limitation 28 | * /server_fqdn#PID_timestamp - cluster-copier worker ID 29 | * ... 30 | * /tables - directory with table tasks 31 | * /cluster.db.table1 - directory of table_hits task 32 | * /partition1 - directory for partition1 33 | * /shards - directory for source cluster shards 34 | * /1 - worker job for the first shard of partition1 of table test.hits 35 | * Contains info about current status (Active or Finished) and worker ID. 36 | * /2 37 | * ... 38 | * /partition_active_workers 39 | * /1 - for each job in /shards a corresponding ephemeral node created in /partition_active_workers 40 | * It is used to detect Abandoned jobs (if there is Active node in /shards and there is no node in 41 | * /partition_active_workers). 42 | * Also, it is used to track active workers in the partition (when we need to refill the partition we do 43 | * not DROP PARTITION while there are active workers) 44 | * /2 45 | * ... 46 | * /is_dirty - the node is set if some worker detected that an error occurred (the INSERT is failed or an Abandoned node is 47 | * detected). If the node appeared workers in this partition should stop and start cleaning and refilling 48 | * partition procedure. 49 | * During this procedure a single 'cleaner' worker is selected. The worker waits for stopping all partition 50 | * workers, removes /shards node, executes DROP PARTITION on each destination node and removes /is_dirty node. 51 | * /cleaner- An ephemeral node used to select 'cleaner' worker. Contains ID of the worker. 52 | * /cluster.db.table2 53 | * ... 54 | */ 55 | 56 | namespace DB 57 | { 58 | 59 | class ClusterCopierApp : public BaseDaemon 60 | { 61 | public: 62 | 63 | void initialize(Poco::Util::Application & self) override; 64 | 65 | void handleHelp(const std::string &, const std::string &); 66 | 67 | void defineOptions(Poco::Util::OptionSet & options) override; 68 | 69 | int main(const std::vector &) override; 70 | 71 | private: 72 | 73 | using Base = BaseDaemon; 74 | 75 | void mainImpl(); 76 | 77 | std::string config_xml_path; 78 | std::string task_path; 79 | std::string log_level = "info"; 80 | bool is_safe_mode = false; 81 | bool is_status_mode = false; 82 | double copy_fault_probability = 0.0; 83 | double move_fault_probability = 0.0; 84 | bool is_help = false; 85 | 86 | UInt64 max_table_tries = 3; 87 | UInt64 max_shard_partition_tries = 3; 88 | UInt64 max_shard_partition_piece_tries_for_alter = 10; 89 | std::chrono::milliseconds retry_delay_ms{1000}; 90 | 91 | bool experimental_use_sample_offset{false}; 92 | 93 | std::string base_dir; 94 | std::string process_path; 95 | std::string process_id; 96 | std::string host_id; 97 | }; 98 | 99 | } 100 | -------------------------------------------------------------------------------- /programs/copier/ClusterPartition.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace DB 7 | { 8 | 9 | /// Contains info about all shards that contain a partition 10 | struct ClusterPartition 11 | { 12 | double elapsed_time_seconds = 0; 13 | UInt64 bytes_copied = 0; 14 | UInt64 rows_copied = 0; 15 | UInt64 blocks_copied = 0; 16 | 17 | UInt64 total_tries = 0; 18 | }; 19 | 20 | using ClusterPartitions = std::map>; 21 | 22 | } 23 | -------------------------------------------------------------------------------- /programs/copier/Internals.cpp: -------------------------------------------------------------------------------- 1 | #include "Internals.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace DB 11 | { 12 | namespace ErrorCodes 13 | { 14 | extern const int BAD_ARGUMENTS; 15 | } 16 | 17 | using ConfigurationPtr = Poco::AutoPtr; 18 | 19 | ConfigurationPtr getConfigurationFromXMLString(const std::string & xml_data) 20 | { 21 | std::stringstream ss(xml_data); // STYLE_CHECK_ALLOW_STD_STRING_STREAM 22 | Poco::XML::InputSource input_source{ss}; 23 | return {new Poco::Util::XMLConfiguration{&input_source}}; 24 | } 25 | 26 | String getQuotedTable(const String & database, const String & table) 27 | { 28 | if (database.empty()) 29 | return backQuoteIfNeed(table); 30 | 31 | return backQuoteIfNeed(database) + "." + backQuoteIfNeed(table); 32 | } 33 | 34 | String getQuotedTable(const DatabaseAndTableName & db_and_table) 35 | { 36 | return getQuotedTable(db_and_table.first, db_and_table.second); 37 | } 38 | 39 | 40 | // Creates AST representing 'ENGINE = Distributed(cluster, db, table, [sharding_key]) 41 | std::shared_ptr createASTStorageDistributed( 42 | const String & cluster_name, const String & database, const String & table, 43 | const ASTPtr & sharding_key_ast) 44 | { 45 | auto args = std::make_shared(); 46 | args->children.emplace_back(std::make_shared(cluster_name)); 47 | args->children.emplace_back(std::make_shared(database)); 48 | args->children.emplace_back(std::make_shared(table)); 49 | if (sharding_key_ast) 50 | args->children.emplace_back(sharding_key_ast); 51 | 52 | auto engine = std::make_shared(); 53 | engine->name = "Distributed"; 54 | engine->arguments = args; 55 | 56 | auto storage = std::make_shared(); 57 | storage->set(storage->engine, engine); 58 | 59 | return storage; 60 | } 61 | 62 | 63 | Block getBlockWithAllStreamData(QueryPipelineBuilder builder) 64 | { 65 | builder.addTransform(std::make_shared( 66 | builder.getHeader(), 67 | std::numeric_limits::max(), 68 | std::numeric_limits::max())); 69 | 70 | auto cur_pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); 71 | Block block; 72 | PullingPipelineExecutor executor(cur_pipeline); 73 | executor.pull(block); 74 | 75 | return block; 76 | } 77 | 78 | bool isExtendedDefinitionStorage(const ASTPtr & storage_ast) 79 | { 80 | const auto & storage = storage_ast->as(); 81 | return storage.partition_by || storage.order_by || storage.sample_by; 82 | } 83 | 84 | ASTPtr extractPartitionKey(const ASTPtr & storage_ast) 85 | { 86 | String storage_str = queryToString(storage_ast); 87 | 88 | const auto & storage = storage_ast->as(); 89 | const auto & engine = storage.engine->as(); 90 | 91 | if (!endsWith(engine.name, "MergeTree")) 92 | { 93 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported engine was specified in {}, only *MergeTree engines are supported", storage_str); 94 | } 95 | 96 | if (isExtendedDefinitionStorage(storage_ast)) 97 | { 98 | if (storage.partition_by) 99 | return storage.partition_by->clone(); 100 | 101 | static const char * all = "all"; 102 | return std::make_shared(Field(all, strlen(all))); 103 | } 104 | else 105 | { 106 | bool is_replicated = startsWith(engine.name, "Replicated"); 107 | size_t min_args = is_replicated ? 3 : 1; 108 | 109 | if (!engine.arguments) 110 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected arguments in {}", storage_str); 111 | 112 | ASTPtr arguments_ast = engine.arguments->clone(); 113 | ASTs & arguments = arguments_ast->children; 114 | 115 | if (arguments.size() < min_args) 116 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected at least {} arguments in {}", min_args, storage_str); 117 | 118 | ASTPtr & month_arg = is_replicated ? arguments[2] : arguments[1]; 119 | return makeASTFunction("toYYYYMM", month_arg->clone()); 120 | } 121 | } 122 | 123 | ASTPtr extractPrimaryKey(const ASTPtr & storage_ast) 124 | { 125 | String storage_str = queryToString(storage_ast); 126 | 127 | const auto & storage = storage_ast->as(); 128 | const auto & engine = storage.engine->as(); 129 | 130 | if (!endsWith(engine.name, "MergeTree")) 131 | { 132 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported engine was specified in {}, only *MergeTree engines are supported", storage_str); 133 | } 134 | 135 | if (!isExtendedDefinitionStorage(storage_ast)) 136 | { 137 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Is not extended deginition storage {} Will be fixed later.", storage_str); 138 | } 139 | 140 | if (storage.primary_key) 141 | return storage.primary_key->clone(); 142 | 143 | return nullptr; 144 | } 145 | 146 | 147 | ASTPtr extractOrderBy(const ASTPtr & storage_ast) 148 | { 149 | String storage_str = queryToString(storage_ast); 150 | 151 | const auto & storage = storage_ast->as(); 152 | const auto & engine = storage.engine->as(); 153 | 154 | if (!endsWith(engine.name, "MergeTree")) 155 | { 156 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported engine was specified in {}, only *MergeTree engines are supported", storage_str); 157 | } 158 | 159 | if (!isExtendedDefinitionStorage(storage_ast)) 160 | { 161 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Is not extended deginition storage {} Will be fixed later.", storage_str); 162 | } 163 | 164 | if (storage.order_by) 165 | return storage.order_by->clone(); 166 | 167 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "ORDER BY cannot be empty"); 168 | } 169 | 170 | /// Wraps only identifiers with backticks. 171 | std::string wrapIdentifiersWithBackticks(const ASTPtr & root) 172 | { 173 | if (auto identifier = std::dynamic_pointer_cast(root)) 174 | return backQuote(identifier->name()); 175 | 176 | if (auto function = std::dynamic_pointer_cast(root)) 177 | return function->name + '(' + wrapIdentifiersWithBackticks(function->arguments) + ')'; 178 | 179 | if (auto expression_list = std::dynamic_pointer_cast(root)) 180 | { 181 | Names function_arguments(expression_list->children.size()); 182 | for (size_t i = 0; i < expression_list->children.size(); ++i) 183 | function_arguments[i] = wrapIdentifiersWithBackticks(expression_list->children[0]); 184 | return boost::algorithm::join(function_arguments, ", "); 185 | } 186 | 187 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Primary key could be represented only as columns or functions from columns."); 188 | } 189 | 190 | 191 | Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast) 192 | { 193 | const auto sorting_key_ast = extractOrderBy(storage_ast); 194 | const auto primary_key_ast = extractPrimaryKey(storage_ast); 195 | 196 | const auto sorting_key_expr_list = extractKeyExpressionList(sorting_key_ast); 197 | const auto primary_key_expr_list = primary_key_ast 198 | ? extractKeyExpressionList(primary_key_ast) : sorting_key_expr_list->clone(); 199 | 200 | /// Maybe we have to handle VersionedCollapsing engine separately. But in our case in looks pointless. 201 | 202 | size_t primary_key_size = primary_key_expr_list->children.size(); 203 | size_t sorting_key_size = sorting_key_expr_list->children.size(); 204 | 205 | if (primary_key_size > sorting_key_size) 206 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Primary key must be a prefix of the sorting key, but its length: " 207 | "{} is greater than the sorting key length: {}", 208 | primary_key_size, sorting_key_size); 209 | 210 | Names primary_key_columns; 211 | NameSet primary_key_columns_set; 212 | 213 | for (size_t i = 0; i < sorting_key_size; ++i) 214 | { 215 | /// Column name could be represented as a f_1(f_2(...f_n(column_name))). 216 | /// Each f_i could take one or more parameters. 217 | /// We will wrap identifiers with backticks to allow non-standard identifier names. 218 | String sorting_key_column = sorting_key_expr_list->children[i]->getColumnName(); 219 | 220 | if (i < primary_key_size) 221 | { 222 | String pk_column = primary_key_expr_list->children[i]->getColumnName(); 223 | if (pk_column != sorting_key_column) 224 | throw Exception(ErrorCodes::BAD_ARGUMENTS, 225 | "Primary key must be a prefix of the sorting key, " 226 | "but the column in the position {} is {}, not {}", i, sorting_key_column, pk_column); 227 | 228 | if (!primary_key_columns_set.emplace(pk_column).second) 229 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Primary key contains duplicate columns"); 230 | 231 | primary_key_columns.push_back(wrapIdentifiersWithBackticks(primary_key_expr_list->children[i])); 232 | } 233 | } 234 | 235 | return primary_key_columns; 236 | } 237 | 238 | bool isReplicatedTableEngine(const ASTPtr & storage_ast) 239 | { 240 | const auto & storage = storage_ast->as(); 241 | const auto & engine = storage.engine->as(); 242 | 243 | if (!endsWith(engine.name, "MergeTree")) 244 | { 245 | String storage_str = queryToString(storage_ast); 246 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported engine was specified in {}, only *MergeTree engines are supported", storage_str); 247 | } 248 | 249 | return startsWith(engine.name, "Replicated"); 250 | } 251 | 252 | ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std::string & local_hostname, UInt8 random) 253 | { 254 | ShardPriority res; 255 | 256 | if (replicas.empty()) 257 | return res; 258 | 259 | res.is_remote = 1; 260 | for (const auto & replica : replicas) 261 | { 262 | if (isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(replica.host_name).front())) 263 | { 264 | res.is_remote = 0; 265 | break; 266 | } 267 | } 268 | 269 | res.hostname_difference = std::numeric_limits::max(); 270 | for (const auto & replica : replicas) 271 | { 272 | size_t difference = getHostNamePrefixDistance(local_hostname, replica.host_name); 273 | res.hostname_difference = std::min(difference, res.hostname_difference); 274 | } 275 | 276 | res.random = random; 277 | return res; 278 | } 279 | 280 | } 281 | -------------------------------------------------------------------------------- /programs/copier/Internals.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | #include 61 | #include 62 | #include 63 | #include 64 | 65 | #include "Aliases.h" 66 | 67 | namespace DB 68 | { 69 | 70 | namespace ErrorCodes 71 | { 72 | extern const int LOGICAL_ERROR; 73 | } 74 | 75 | 76 | ConfigurationPtr getConfigurationFromXMLString(const std::string & xml_data); 77 | 78 | String getQuotedTable(const String & database, const String & table); 79 | 80 | String getQuotedTable(const DatabaseAndTableName & db_and_table); 81 | 82 | 83 | enum class TaskState 84 | { 85 | Started = 0, 86 | Finished, 87 | Unknown 88 | }; 89 | 90 | /// Used to mark status of shard partition tasks 91 | struct TaskStateWithOwner 92 | { 93 | TaskStateWithOwner() = default; 94 | 95 | TaskStateWithOwner(TaskState state_, const String & owner_) : state(state_), owner(owner_) {} 96 | 97 | TaskState state{TaskState::Unknown}; 98 | String owner; 99 | 100 | static String getData(TaskState state, const String &owner) 101 | { 102 | return TaskStateWithOwner(state, owner).toString(); 103 | } 104 | 105 | String toString() const 106 | { 107 | WriteBufferFromOwnString wb; 108 | wb << static_cast(state) << "\n" << escape << owner; 109 | return wb.str(); 110 | } 111 | 112 | static TaskStateWithOwner fromString(const String & data) 113 | { 114 | ReadBufferFromString rb(data); 115 | TaskStateWithOwner res; 116 | UInt32 state; 117 | 118 | rb >> state >> "\n" >> escape >> res.owner; 119 | 120 | if (state >= static_cast(TaskState::Unknown)) 121 | throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown state {}", data); 122 | 123 | res.state = static_cast(state); 124 | return res; 125 | } 126 | }; 127 | 128 | 129 | struct ShardPriority 130 | { 131 | UInt8 is_remote = 1; 132 | size_t hostname_difference = 0; 133 | UInt8 random = 0; 134 | 135 | static bool greaterPriority(const ShardPriority & current, const ShardPriority & other) 136 | { 137 | return std::forward_as_tuple(current.is_remote, current.hostname_difference, current.random) 138 | < std::forward_as_tuple(other.is_remote, other.hostname_difference, other.random); 139 | } 140 | }; 141 | 142 | /// Execution status of a task. 143 | /// Is used for: partition copying task status, partition piece copying task status, partition moving task status. 144 | enum class TaskStatus 145 | { 146 | Active, 147 | Finished, 148 | Error, 149 | }; 150 | 151 | struct MultiTransactionInfo 152 | { 153 | int32_t code; 154 | Coordination::Requests requests; 155 | Coordination::Responses responses; 156 | }; 157 | 158 | // Creates AST representing 'ENGINE = Distributed(cluster, db, table, [sharding_key]) 159 | std::shared_ptr createASTStorageDistributed( 160 | const String & cluster_name, const String & database, const String & table, 161 | const ASTPtr & sharding_key_ast = nullptr); 162 | 163 | Block getBlockWithAllStreamData(QueryPipelineBuilder builder); 164 | 165 | bool isExtendedDefinitionStorage(const ASTPtr & storage_ast); 166 | 167 | ASTPtr extractPartitionKey(const ASTPtr & storage_ast); 168 | 169 | /* 170 | * Choosing a Primary Key that Differs from the Sorting Key 171 | * It is possible to specify a primary key (an expression with values that are written in the index file for each mark) 172 | * that is different from the sorting key (an expression for sorting the rows in data parts). 173 | * In this case the primary key expression tuple must be a prefix of the sorting key expression tuple. 174 | * This feature is helpful when using the SummingMergeTree and AggregatingMergeTree table engines. 175 | * In a common case when using these engines, the table has two types of columns: dimensions and measures. 176 | * Typical queries aggregate values of measure columns with arbitrary GROUP BY and filtering by dimensions. 177 | * Because SummingMergeTree and AggregatingMergeTree aggregate rows with the same value of the sorting key, 178 | * it is natural to add all dimensions to it. As a result, the key expression consists of a long list of columns 179 | * and this list must be frequently updated with newly added dimensions. 180 | * In this case it makes sense to leave only a few columns in the primary key that will provide efficient 181 | * range scans and add the remaining dimension columns to the sorting key tuple. 182 | * ALTER of the sorting key is a lightweight operation because when a new column is simultaneously added t 183 | * o the table and to the sorting key, existing data parts don't need to be changed. 184 | * Since the old sorting key is a prefix of the new sorting key and there is no data in the newly added column, 185 | * the data is sorted by both the old and new sorting keys at the moment of table modification. 186 | * 187 | * */ 188 | ASTPtr extractPrimaryKey(const ASTPtr & storage_ast); 189 | 190 | ASTPtr extractOrderBy(const ASTPtr & storage_ast); 191 | 192 | Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast); 193 | 194 | bool isReplicatedTableEngine(const ASTPtr & storage_ast); 195 | 196 | ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std::string & local_hostname, UInt8 random); 197 | 198 | } 199 | -------------------------------------------------------------------------------- /programs/copier/ShardPartition.cpp: -------------------------------------------------------------------------------- 1 | #include "ShardPartition.h" 2 | 3 | #include "TaskShard.h" 4 | #include "TaskTable.h" 5 | 6 | namespace DB 7 | { 8 | 9 | ShardPartition::ShardPartition(TaskShard & parent, String name_quoted_, size_t number_of_splits) 10 | : task_shard(parent) 11 | , name(std::move(name_quoted_)) 12 | { 13 | pieces.reserve(number_of_splits); 14 | } 15 | 16 | String ShardPartition::getPartitionCleanStartPath() const 17 | { 18 | return getPartitionPath() + "/clean_start"; 19 | } 20 | 21 | String ShardPartition::getPartitionPieceCleanStartPath(size_t current_piece_number) const 22 | { 23 | assert(current_piece_number < task_shard.task_table.number_of_splits); 24 | return getPartitionPiecePath(current_piece_number) + "/clean_start"; 25 | } 26 | 27 | String ShardPartition::getPartitionPath() const 28 | { 29 | return task_shard.task_table.getPartitionPath(name); 30 | } 31 | 32 | String ShardPartition::getPartitionPiecePath(size_t current_piece_number) const 33 | { 34 | assert(current_piece_number < task_shard.task_table.number_of_splits); 35 | return task_shard.task_table.getPartitionPiecePath(name, current_piece_number); 36 | } 37 | 38 | String ShardPartition::getShardStatusPath() const 39 | { 40 | // schema: //tables///shards/ 41 | // e.g. /root/table_test.hits/201701/shards/1 42 | return getPartitionShardsPath() + "/" + toString(task_shard.numberInCluster()); 43 | } 44 | 45 | String ShardPartition::getPartitionShardsPath() const 46 | { 47 | return getPartitionPath() + "/shards"; 48 | } 49 | 50 | String ShardPartition::getPartitionActiveWorkersPath() const 51 | { 52 | return getPartitionPath() + "/partition_active_workers"; 53 | } 54 | 55 | String ShardPartition::getActiveWorkerPath() const 56 | { 57 | return getPartitionActiveWorkersPath() + "/" + toString(task_shard.numberInCluster()); 58 | } 59 | 60 | String ShardPartition::getCommonPartitionIsDirtyPath() const 61 | { 62 | return getPartitionPath() + "/is_dirty"; 63 | } 64 | 65 | String ShardPartition::getCommonPartitionIsCleanedPath() const 66 | { 67 | return getCommonPartitionIsDirtyPath() + "/cleaned"; 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /programs/copier/ShardPartition.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "ShardPartitionPiece.h" 4 | 5 | #include 6 | 7 | #include 8 | 9 | namespace DB 10 | { 11 | 12 | struct TaskShard; 13 | 14 | /// Just destination partition of a shard 15 | /// I don't know what this comment means. 16 | /// In short, when we discovered what shards contain currently processing partition, 17 | /// This class describes a partition (name) that is stored on the shard (parent). 18 | struct ShardPartition 19 | { 20 | ShardPartition(TaskShard &parent, String name_quoted_, size_t number_of_splits = 10); 21 | 22 | String getPartitionPath() const; 23 | 24 | String getPartitionPiecePath(size_t current_piece_number) const; 25 | 26 | String getPartitionCleanStartPath() const; 27 | 28 | String getPartitionPieceCleanStartPath(size_t current_piece_number) const; 29 | 30 | String getCommonPartitionIsDirtyPath() const; 31 | 32 | String getCommonPartitionIsCleanedPath() const; 33 | 34 | String getPartitionActiveWorkersPath() const; 35 | 36 | String getActiveWorkerPath() const; 37 | 38 | String getPartitionShardsPath() const; 39 | 40 | String getShardStatusPath() const; 41 | 42 | /// What partition pieces are present in current shard. 43 | /// FYI: Piece is a part of partition which has modulo equals to concrete constant (less than number_of_splits obliously) 44 | /// For example SELECT ... from ... WHERE partition=current_partition AND cityHash64(*) == const; 45 | /// Absent pieces have field is_absent_piece equals to true. 46 | PartitionPieces pieces; 47 | 48 | TaskShard & task_shard; 49 | String name; 50 | }; 51 | 52 | using TasksPartition = std::map>; 53 | 54 | } 55 | -------------------------------------------------------------------------------- /programs/copier/ShardPartitionPiece.cpp: -------------------------------------------------------------------------------- 1 | #include "ShardPartitionPiece.h" 2 | 3 | #include "ShardPartition.h" 4 | #include "TaskShard.h" 5 | 6 | #include 7 | 8 | namespace DB 9 | { 10 | 11 | ShardPartitionPiece::ShardPartitionPiece(ShardPartition & parent, size_t current_piece_number_, bool is_present_piece_) 12 | : is_absent_piece(!is_present_piece_) 13 | , current_piece_number(current_piece_number_) 14 | , shard_partition(parent) 15 | { 16 | } 17 | 18 | String ShardPartitionPiece::getPartitionPiecePath() const 19 | { 20 | return shard_partition.getPartitionPath() + "/piece_" + toString(current_piece_number); 21 | } 22 | 23 | String ShardPartitionPiece::getPartitionPieceCleanStartPath() const 24 | { 25 | return getPartitionPiecePath() + "/clean_start"; 26 | } 27 | 28 | String ShardPartitionPiece::getPartitionPieceIsDirtyPath() const 29 | { 30 | return getPartitionPiecePath() + "/is_dirty"; 31 | } 32 | 33 | String ShardPartitionPiece::getPartitionPieceIsCleanedPath() const 34 | { 35 | return getPartitionPieceIsDirtyPath() + "/cleaned"; 36 | } 37 | 38 | String ShardPartitionPiece::getPartitionPieceActiveWorkersPath() const 39 | { 40 | return getPartitionPiecePath() + "/partition_piece_active_workers"; 41 | } 42 | 43 | String ShardPartitionPiece::getActiveWorkerPath() const 44 | { 45 | return getPartitionPieceActiveWorkersPath() + "/" + toString(shard_partition.task_shard.numberInCluster()); 46 | } 47 | 48 | /// On what shards do we have current partition. 49 | String ShardPartitionPiece::getPartitionPieceShardsPath() const 50 | { 51 | return getPartitionPiecePath() + "/shards"; 52 | } 53 | 54 | String ShardPartitionPiece::getShardStatusPath() const 55 | { 56 | return getPartitionPieceShardsPath() + "/" + toString(shard_partition.task_shard.numberInCluster()); 57 | } 58 | 59 | String ShardPartitionPiece::getPartitionPieceCleanerPath() const 60 | { 61 | return getPartitionPieceIsDirtyPath() + "/cleaner"; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /programs/copier/ShardPartitionPiece.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | 7 | namespace DB 8 | { 9 | 10 | struct ShardPartition; 11 | 12 | struct ShardPartitionPiece 13 | { 14 | ShardPartitionPiece(ShardPartition & parent, size_t current_piece_number_, bool is_present_piece_); 15 | 16 | String getPartitionPiecePath() const; 17 | 18 | String getPartitionPieceCleanStartPath() const; 19 | 20 | String getPartitionPieceIsDirtyPath() const; 21 | 22 | String getPartitionPieceIsCleanedPath() const; 23 | 24 | String getPartitionPieceActiveWorkersPath() const; 25 | 26 | String getActiveWorkerPath() const ; 27 | 28 | /// On what shards do we have current partition. 29 | String getPartitionPieceShardsPath() const; 30 | 31 | String getShardStatusPath() const; 32 | 33 | String getPartitionPieceCleanerPath() const; 34 | 35 | bool is_absent_piece; 36 | const size_t current_piece_number; 37 | 38 | ShardPartition & shard_partition; 39 | }; 40 | 41 | using PartitionPieces = std::vector; 42 | 43 | } 44 | -------------------------------------------------------------------------------- /programs/copier/StatusAccumulator.cpp: -------------------------------------------------------------------------------- 1 | #include "StatusAccumulator.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | namespace DB 11 | { 12 | 13 | StatusAccumulator::MapPtr StatusAccumulator::fromJSON(String state_json) 14 | { 15 | Poco::JSON::Parser parser; 16 | auto state = parser.parse(state_json).extract(); 17 | MapPtr result_ptr = std::make_shared(); 18 | for (const auto & table_name : state->getNames()) 19 | { 20 | auto table_status_json = state->getValue(table_name); 21 | auto table_status = parser.parse(table_status_json).extract(); 22 | /// Map entry will be created if it is absent 23 | auto & map_table_status = (*result_ptr)[table_name]; 24 | map_table_status.all_partitions_count += table_status->getValue("all_partitions_count"); 25 | map_table_status.processed_partitions_count += table_status->getValue("processed_partitions_count"); 26 | } 27 | return result_ptr; 28 | } 29 | 30 | String StatusAccumulator::serializeToJSON(MapPtr statuses) 31 | { 32 | Poco::JSON::Object result_json; 33 | for (const auto & [table_name, table_status] : *statuses) 34 | { 35 | Poco::JSON::Object status_json; 36 | status_json.set("all_partitions_count", table_status.all_partitions_count); 37 | status_json.set("processed_partitions_count", table_status.processed_partitions_count); 38 | 39 | result_json.set(table_name, status_json); 40 | } 41 | std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM 42 | oss.exceptions(std::ios::failbit); 43 | Poco::JSON::Stringifier::stringify(result_json, oss); 44 | auto result = oss.str(); 45 | return result; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /programs/copier/StatusAccumulator.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | namespace DB 9 | { 10 | 11 | class StatusAccumulator 12 | { 13 | public: 14 | struct TableStatus 15 | { 16 | size_t all_partitions_count; 17 | size_t processed_partitions_count; 18 | }; 19 | 20 | using Map = std::unordered_map; 21 | using MapPtr = std::shared_ptr; 22 | 23 | static MapPtr fromJSON(String state_json); 24 | static String serializeToJSON(MapPtr statuses); 25 | }; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /programs/copier/TaskCluster.cpp: -------------------------------------------------------------------------------- 1 | #include "TaskCluster.h" 2 | 3 | namespace DB 4 | { 5 | 6 | namespace ErrorCodes 7 | { 8 | extern const int BAD_ARGUMENTS; 9 | } 10 | 11 | TaskCluster::TaskCluster(const String & task_zookeeper_path_, const String & default_local_database_) 12 | : task_zookeeper_path(task_zookeeper_path_) 13 | , default_local_database(default_local_database_) 14 | {} 15 | 16 | void DB::TaskCluster::loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key) 17 | { 18 | String prefix = base_key.empty() ? "" : base_key + "."; 19 | 20 | clusters_prefix = prefix + "remote_servers"; 21 | if (!config.has(clusters_prefix)) 22 | throw Exception(ErrorCodes::BAD_ARGUMENTS, "You should specify list of clusters in {}", clusters_prefix); 23 | 24 | Poco::Util::AbstractConfiguration::Keys tables_keys; 25 | config.keys(prefix + "tables", tables_keys); 26 | 27 | for (const auto & table_key : tables_keys) 28 | { 29 | table_tasks.emplace_back(*this, config, prefix + "tables", table_key); 30 | } 31 | } 32 | 33 | void DB::TaskCluster::reloadSettings(const Poco::Util::AbstractConfiguration & config, const String & base_key) 34 | { 35 | String prefix = base_key.empty() ? "" : base_key + "."; 36 | 37 | max_workers = config.getUInt64(prefix + "max_workers"); 38 | 39 | settings_common = Settings(); 40 | if (config.has(prefix + "settings")) 41 | settings_common.loadSettingsFromConfig(prefix + "settings", config); 42 | 43 | settings_common.prefer_localhost_replica = false; 44 | 45 | settings_pull = settings_common; 46 | if (config.has(prefix + "settings_pull")) 47 | settings_pull.loadSettingsFromConfig(prefix + "settings_pull", config); 48 | 49 | settings_push = settings_common; 50 | if (config.has(prefix + "settings_push")) 51 | settings_push.loadSettingsFromConfig(prefix + "settings_push", config); 52 | 53 | auto set_default_value = [] (auto && setting, auto && default_value) 54 | { 55 | setting = setting.changed ? setting.value : default_value; 56 | }; 57 | 58 | /// Override important settings 59 | settings_pull.readonly = 1; 60 | settings_pull.prefer_localhost_replica = false; 61 | settings_push.distributed_foreground_insert = true; 62 | settings_push.prefer_localhost_replica = false; 63 | 64 | set_default_value(settings_pull.load_balancing, LoadBalancing::NEAREST_HOSTNAME); 65 | set_default_value(settings_pull.max_threads, 1); 66 | set_default_value(settings_pull.max_block_size, 8192UL); 67 | set_default_value(settings_pull.preferred_block_size_bytes, 0); 68 | 69 | set_default_value(settings_push.distributed_background_insert_timeout, 0); 70 | set_default_value(settings_push.alter_sync, 2); 71 | } 72 | 73 | } 74 | 75 | -------------------------------------------------------------------------------- /programs/copier/TaskCluster.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "TaskTable.h" 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | 12 | namespace DB 13 | { 14 | 15 | struct TaskCluster 16 | { 17 | TaskCluster(const String & task_zookeeper_path_, const String & default_local_database_); 18 | 19 | void loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key = ""); 20 | 21 | /// Set (or update) settings and max_workers param 22 | void reloadSettings(const Poco::Util::AbstractConfiguration & config, const String & base_key = ""); 23 | 24 | /// Base node for all tasks. Its structure: 25 | /// workers/ - directory with active workers (amount of them is less or equal max_workers) 26 | /// description - node with task configuration 27 | /// table_table1/ - directories with per-partition copying status 28 | String task_zookeeper_path; 29 | 30 | /// Database used to create temporary Distributed tables 31 | String default_local_database; 32 | 33 | /// Limits number of simultaneous workers 34 | UInt64 max_workers = 0; 35 | 36 | /// Base settings for pull and push 37 | Settings settings_common; 38 | /// Settings used to fetch data 39 | Settings settings_pull; 40 | /// Settings used to insert data 41 | Settings settings_push; 42 | 43 | String clusters_prefix; 44 | 45 | /// Subtasks 46 | TasksTable table_tasks; 47 | 48 | pcg64 random_engine; 49 | }; 50 | 51 | } 52 | -------------------------------------------------------------------------------- /programs/copier/TaskShard.cpp: -------------------------------------------------------------------------------- 1 | #include "TaskShard.h" 2 | 3 | #include "TaskTable.h" 4 | 5 | namespace DB 6 | { 7 | 8 | TaskShard::TaskShard(TaskTable & parent, const Cluster::ShardInfo & info_) 9 | : task_table(parent) 10 | , info(info_) 11 | { 12 | list_of_split_tables_on_shard.assign(task_table.number_of_splits, DatabaseAndTableName()); 13 | } 14 | 15 | UInt32 TaskShard::numberInCluster() const 16 | { 17 | return info.shard_num; 18 | } 19 | 20 | UInt32 TaskShard::indexInCluster() const 21 | { 22 | return info.shard_num - 1; 23 | } 24 | 25 | String DB::TaskShard::getDescription() const 26 | { 27 | return fmt::format("N{} (having a replica {}, pull table {} of cluster {}", 28 | numberInCluster(), getHostNameExample(), getQuotedTable(task_table.table_pull), task_table.cluster_pull_name); 29 | } 30 | 31 | String DB::TaskShard::getHostNameExample() const 32 | { 33 | const auto & replicas = task_table.cluster_pull->getShardsAddresses().at(indexInCluster()); 34 | return replicas.at(0).readableString(); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /programs/copier/TaskShard.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Aliases.h" 4 | #include "Internals.h" 5 | #include "ClusterPartition.h" 6 | #include "ShardPartition.h" 7 | 8 | 9 | namespace DB 10 | { 11 | 12 | struct TaskTable; 13 | 14 | struct TaskShard 15 | { 16 | TaskShard(TaskTable & parent, const Cluster::ShardInfo & info_); 17 | 18 | TaskTable & task_table; 19 | 20 | Cluster::ShardInfo info; 21 | 22 | UInt32 numberInCluster() const; 23 | 24 | UInt32 indexInCluster() const; 25 | 26 | String getDescription() const; 27 | 28 | String getHostNameExample() const; 29 | 30 | /// Used to sort clusters by their proximity 31 | ShardPriority priority; 32 | 33 | /// Column with unique destination partitions (computed from engine_push_partition_key expr.) in the shard 34 | ColumnWithTypeAndName partition_key_column; 35 | 36 | /// There is a task for each destination partition 37 | TasksPartition partition_tasks; 38 | 39 | /// Which partitions have been checked for existence 40 | /// If some partition from this lists is exists, it is in partition_tasks 41 | std::set checked_partitions; 42 | 43 | /// Last CREATE TABLE query of the table of the shard 44 | ASTPtr current_pull_table_create_query; 45 | ASTPtr current_push_table_create_query; 46 | 47 | /// Internal distributed tables 48 | DatabaseAndTableName table_read_shard; 49 | DatabaseAndTableName main_table_split_shard; 50 | ListOfDatabasesAndTableNames list_of_split_tables_on_shard; 51 | }; 52 | 53 | using TaskShardPtr = std::shared_ptr; 54 | using TasksShard = std::vector; 55 | 56 | } 57 | -------------------------------------------------------------------------------- /programs/copier/TaskTable.cpp: -------------------------------------------------------------------------------- 1 | #include "TaskTable.h" 2 | 3 | #include "ClusterPartition.h" 4 | #include "TaskCluster.h" 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | 12 | namespace DB 13 | { 14 | namespace ErrorCodes 15 | { 16 | extern const int UNKNOWN_ELEMENT_IN_CONFIG; 17 | extern const int LOGICAL_ERROR; 18 | } 19 | 20 | TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConfiguration & config, 21 | const String & prefix_, const String & table_key) 22 | : task_cluster(parent) 23 | { 24 | String table_prefix = prefix_ + "." + table_key + "."; 25 | 26 | name_in_config = table_key; 27 | 28 | number_of_splits = config.getUInt64(table_prefix + "number_of_splits", 3); 29 | 30 | allow_to_copy_alias_and_materialized_columns = config.getBool(table_prefix + "allow_to_copy_alias_and_materialized_columns", false); 31 | allow_to_drop_target_partitions = config.getBool(table_prefix + "allow_to_drop_target_partitions", false); 32 | 33 | cluster_pull_name = config.getString(table_prefix + "cluster_pull"); 34 | cluster_push_name = config.getString(table_prefix + "cluster_push"); 35 | 36 | table_pull.first = config.getString(table_prefix + "database_pull"); 37 | table_pull.second = config.getString(table_prefix + "table_pull"); 38 | 39 | table_push.first = config.getString(table_prefix + "database_push"); 40 | table_push.second = config.getString(table_prefix + "table_push"); 41 | 42 | /// Used as node name in ZooKeeper 43 | table_id = escapeForFileName(cluster_push_name) 44 | + "." + escapeForFileName(table_push.first) 45 | + "." + escapeForFileName(table_push.second); 46 | 47 | engine_push_str = config.getString(table_prefix + "engine", "rand()"); 48 | 49 | { 50 | ParserStorage parser_storage{ParserStorage::TABLE_ENGINE}; 51 | engine_push_ast = parseQuery(parser_storage, engine_push_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); 52 | engine_push_partition_key_ast = extractPartitionKey(engine_push_ast); 53 | primary_key_comma_separated = boost::algorithm::join(extractPrimaryKeyColumnNames(engine_push_ast), ", "); 54 | is_replicated_table = isReplicatedTableEngine(engine_push_ast); 55 | } 56 | 57 | sharding_key_str = config.getString(table_prefix + "sharding_key"); 58 | 59 | auxiliary_engine_split_asts.reserve(number_of_splits); 60 | { 61 | ParserExpressionWithOptionalAlias parser_expression(false); 62 | sharding_key_ast = parseQuery(parser_expression, sharding_key_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); 63 | main_engine_split_ast = createASTStorageDistributed(cluster_push_name, table_push.first, table_push.second, 64 | sharding_key_ast); 65 | 66 | for (const auto piece_number : collections::range(0, number_of_splits)) 67 | { 68 | auxiliary_engine_split_asts.emplace_back 69 | ( 70 | createASTStorageDistributed(cluster_push_name, table_push.first, 71 | table_push.second + "_piece_" + toString(piece_number), sharding_key_ast) 72 | ); 73 | } 74 | } 75 | 76 | where_condition_str = config.getString(table_prefix + "where_condition", ""); 77 | if (!where_condition_str.empty()) 78 | { 79 | ParserExpressionWithOptionalAlias parser_expression(false); 80 | where_condition_ast = parseQuery(parser_expression, where_condition_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); 81 | 82 | // Will use canonical expression form 83 | where_condition_str = queryToString(where_condition_ast); 84 | } 85 | 86 | String enabled_partitions_prefix = table_prefix + "enabled_partitions"; 87 | has_enabled_partitions = config.has(enabled_partitions_prefix); 88 | 89 | if (has_enabled_partitions) 90 | { 91 | Strings keys; 92 | config.keys(enabled_partitions_prefix, keys); 93 | 94 | if (keys.empty()) 95 | { 96 | /// Parse list of partition from space-separated string 97 | String partitions_str = config.getString(table_prefix + "enabled_partitions"); 98 | boost::trim_if(partitions_str, isWhitespaceASCII); 99 | boost::split(enabled_partitions, partitions_str, isWhitespaceASCII, boost::token_compress_on); 100 | } 101 | else 102 | { 103 | /// Parse sequence of ... 104 | for (const String &key : keys) 105 | { 106 | if (!startsWith(key, "partition")) 107 | throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown key {} in {}", key, enabled_partitions_prefix); 108 | 109 | enabled_partitions.emplace_back(config.getString(enabled_partitions_prefix + "." + key)); 110 | } 111 | } 112 | 113 | std::copy(enabled_partitions.begin(), enabled_partitions.end(), std::inserter(enabled_partitions_set, enabled_partitions_set.begin())); 114 | } 115 | } 116 | 117 | 118 | String TaskTable::getPartitionPath(const String & partition_name) const 119 | { 120 | return task_cluster.task_zookeeper_path // root 121 | + "/tables/" + table_id // tables/dst_cluster.merge.hits 122 | + "/" + escapeForFileName(partition_name); // 201701 123 | } 124 | 125 | String TaskTable::getPartitionAttachIsActivePath(const String & partition_name) const 126 | { 127 | return getPartitionPath(partition_name) + "/attach_active"; 128 | } 129 | 130 | String TaskTable::getPartitionAttachIsDonePath(const String & partition_name) const 131 | { 132 | return getPartitionPath(partition_name) + "/attach_is_done"; 133 | } 134 | 135 | String TaskTable::getPartitionPiecePath(const String & partition_name, size_t piece_number) const 136 | { 137 | assert(piece_number < number_of_splits); 138 | return getPartitionPath(partition_name) + "/piece_" + toString(piece_number); // 1...number_of_splits 139 | } 140 | 141 | String TaskTable::getCertainPartitionIsDirtyPath(const String &partition_name) const 142 | { 143 | return getPartitionPath(partition_name) + "/is_dirty"; 144 | } 145 | 146 | String TaskTable::getCertainPartitionPieceIsDirtyPath(const String & partition_name, const size_t piece_number) const 147 | { 148 | return getPartitionPiecePath(partition_name, piece_number) + "/is_dirty"; 149 | } 150 | 151 | String TaskTable::getCertainPartitionIsCleanedPath(const String & partition_name) const 152 | { 153 | return getCertainPartitionIsDirtyPath(partition_name) + "/cleaned"; 154 | } 155 | 156 | String TaskTable::getCertainPartitionPieceIsCleanedPath(const String & partition_name, const size_t piece_number) const 157 | { 158 | return getCertainPartitionPieceIsDirtyPath(partition_name, piece_number) + "/cleaned"; 159 | } 160 | 161 | String TaskTable::getCertainPartitionTaskStatusPath(const String & partition_name) const 162 | { 163 | return getPartitionPath(partition_name) + "/shards"; 164 | } 165 | 166 | String TaskTable::getCertainPartitionPieceTaskStatusPath(const String & partition_name, const size_t piece_number) const 167 | { 168 | return getPartitionPiecePath(partition_name, piece_number) + "/shards"; 169 | } 170 | 171 | bool TaskTable::isReplicatedTable() const 172 | { 173 | return is_replicated_table; 174 | } 175 | 176 | String TaskTable::getStatusAllPartitionCount() const 177 | { 178 | return task_cluster.task_zookeeper_path + "/status/all_partitions_count"; 179 | } 180 | 181 | String TaskTable::getStatusProcessedPartitionsCount() const 182 | { 183 | return task_cluster.task_zookeeper_path + "/status/processed_partitions_count"; 184 | } 185 | 186 | ASTPtr TaskTable::rewriteReplicatedCreateQueryToPlain() const 187 | { 188 | ASTPtr prev_engine_push_ast = engine_push_ast->clone(); 189 | 190 | auto & new_storage_ast = prev_engine_push_ast->as(); 191 | auto & new_engine_ast = new_storage_ast.engine->as(); 192 | 193 | /// Remove "Replicated" from name 194 | new_engine_ast.name = new_engine_ast.name.substr(10); 195 | 196 | if (new_engine_ast.arguments) 197 | { 198 | auto & replicated_table_arguments = new_engine_ast.arguments->children; 199 | 200 | 201 | /// In some cases of Atomic database engine usage ReplicatedMergeTree tables 202 | /// could be created without arguments. 203 | if (!replicated_table_arguments.empty()) 204 | { 205 | /// Delete first two arguments of Replicated...MergeTree() table. 206 | replicated_table_arguments.erase(replicated_table_arguments.begin()); 207 | replicated_table_arguments.erase(replicated_table_arguments.begin()); 208 | } 209 | } 210 | 211 | return new_storage_ast.clone(); 212 | } 213 | 214 | ClusterPartition & TaskTable::getClusterPartition(const String & partition_name) 215 | { 216 | auto it = cluster_partitions.find(partition_name); 217 | if (it == cluster_partitions.end()) 218 | throw Exception(ErrorCodes::LOGICAL_ERROR, "There are no cluster partition {} in {}", partition_name, table_id); 219 | return it->second; 220 | } 221 | 222 | } 223 | -------------------------------------------------------------------------------- /programs/copier/TaskTable.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Aliases.h" 4 | #include "TaskShard.h" 5 | 6 | 7 | namespace DB 8 | { 9 | 10 | struct ClusterPartition; 11 | struct TaskCluster; 12 | 13 | struct TaskTable 14 | { 15 | TaskTable(TaskCluster & parent, const Poco::Util::AbstractConfiguration & config, const String & prefix, const String & table_key); 16 | 17 | TaskCluster & task_cluster; 18 | 19 | /// These functions used in checkPartitionIsDone() or checkPartitionPieceIsDone() 20 | /// They are implemented here not to call task_table.tasks_shard[partition_name].second.pieces[current_piece_number] etc. 21 | 22 | String getPartitionPath(const String & partition_name) const; 23 | 24 | String getPartitionAttachIsActivePath(const String & partition_name) const; 25 | 26 | String getPartitionAttachIsDonePath(const String & partition_name) const; 27 | 28 | String getPartitionPiecePath(const String & partition_name, size_t piece_number) const; 29 | 30 | String getCertainPartitionIsDirtyPath(const String & partition_name) const; 31 | 32 | String getCertainPartitionPieceIsDirtyPath(const String & partition_name, size_t piece_number) const; 33 | 34 | String getCertainPartitionIsCleanedPath(const String & partition_name) const; 35 | 36 | String getCertainPartitionPieceIsCleanedPath(const String & partition_name, size_t piece_number) const; 37 | 38 | String getCertainPartitionTaskStatusPath(const String & partition_name) const; 39 | 40 | String getCertainPartitionPieceTaskStatusPath(const String & partition_name, size_t piece_number) const; 41 | 42 | bool isReplicatedTable() const; 43 | 44 | /// These nodes are used for check-status option 45 | String getStatusAllPartitionCount() const; 46 | String getStatusProcessedPartitionsCount() const; 47 | 48 | /// Partitions will be split into number-of-splits pieces. 49 | /// Each piece will be copied independently. (10 by default) 50 | size_t number_of_splits; 51 | 52 | bool allow_to_copy_alias_and_materialized_columns{false}; 53 | bool allow_to_drop_target_partitions{false}; 54 | 55 | String name_in_config; 56 | 57 | /// Used as task ID 58 | String table_id; 59 | 60 | /// Column names in primary key 61 | String primary_key_comma_separated; 62 | 63 | /// Source cluster and table 64 | String cluster_pull_name; 65 | DatabaseAndTableName table_pull; 66 | 67 | /// Destination cluster and table 68 | String cluster_push_name; 69 | DatabaseAndTableName table_push; 70 | 71 | /// Storage of destination table 72 | /// (tables that are stored on each shard of target cluster) 73 | String engine_push_str; 74 | ASTPtr engine_push_ast; 75 | ASTPtr engine_push_partition_key_ast; 76 | 77 | /// First argument of Replicated...MergeTree() 78 | String engine_push_zk_path; 79 | bool is_replicated_table; 80 | 81 | ASTPtr rewriteReplicatedCreateQueryToPlain() const; 82 | 83 | /* 84 | * A Distributed table definition used to split data 85 | * Distributed table will be created on each shard of default 86 | * cluster to perform data copying and resharding 87 | * */ 88 | String sharding_key_str; 89 | ASTPtr sharding_key_ast; 90 | ASTPtr main_engine_split_ast; 91 | 92 | /* 93 | * To copy partition piece form one cluster to another we have to use Distributed table. 94 | * In case of usage separate table (engine_push) for each partition piece, 95 | * we have to use many Distributed tables. 96 | * */ 97 | ASTs auxiliary_engine_split_asts; 98 | 99 | /// Additional WHERE expression to filter input data 100 | String where_condition_str; 101 | ASTPtr where_condition_ast; 102 | 103 | /// Resolved clusters 104 | ClusterPtr cluster_pull; 105 | ClusterPtr cluster_push; 106 | 107 | /// Filter partitions that should be copied 108 | bool has_enabled_partitions = false; 109 | Strings enabled_partitions; 110 | NameSet enabled_partitions_set; 111 | 112 | /** 113 | * Prioritized list of shards 114 | * all_shards contains information about all shards in the table. 115 | * So we have to check whether particular shard have current partition or not while processing. 116 | */ 117 | TasksShard all_shards; 118 | TasksShard local_shards; 119 | 120 | /// All partitions of the current table. 121 | ClusterPartitions cluster_partitions; 122 | NameSet finished_cluster_partitions; 123 | 124 | /// Partition names to process in user-specified order 125 | Strings ordered_partition_names; 126 | 127 | ClusterPartition & getClusterPartition(const String & partition_name); 128 | 129 | Stopwatch watch; 130 | UInt64 bytes_copied = 0; 131 | UInt64 rows_copied = 0; 132 | 133 | template 134 | void initShards(RandomEngine &&random_engine); 135 | }; 136 | 137 | using TasksTable = std::list; 138 | 139 | 140 | template 141 | inline void TaskTable::initShards(RandomEngine && random_engine) 142 | { 143 | const String & fqdn_name = getFQDNOrHostName(); 144 | std::uniform_int_distribution get_urand(0, std::numeric_limits::max()); 145 | 146 | // Compute the priority 147 | for (const auto & shard_info : cluster_pull->getShardsInfo()) 148 | { 149 | TaskShardPtr task_shard = std::make_shared(*this, shard_info); 150 | const auto & replicas = cluster_pull->getShardsAddresses().at(task_shard->indexInCluster()); 151 | task_shard->priority = getReplicasPriority(replicas, fqdn_name, get_urand(random_engine)); 152 | 153 | all_shards.emplace_back(task_shard); 154 | } 155 | 156 | // Sort by priority 157 | std::sort(all_shards.begin(), all_shards.end(), 158 | [](const TaskShardPtr & lhs, const TaskShardPtr & rhs) 159 | { 160 | return ShardPriority::greaterPriority(lhs->priority, rhs->priority); 161 | }); 162 | 163 | // Cut local shards 164 | auto it_first_remote = std::lower_bound(all_shards.begin(), all_shards.end(), 1, 165 | [](const TaskShardPtr & lhs, UInt8 is_remote) 166 | { 167 | return lhs->priority.is_remote < is_remote; 168 | }); 169 | 170 | local_shards.assign(all_shards.begin(), it_first_remote); 171 | } 172 | 173 | } 174 | -------------------------------------------------------------------------------- /programs/copier/ZooKeeperStaff.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /** Allows to compare two incremental counters of type UInt32 in presence of possible overflow. 4 | * We assume that we compare values that are not too far away. 5 | * For example, when we increment 0xFFFFFFFF, we get 0. So, 0xFFFFFFFF is less than 0. 6 | */ 7 | class WrappingUInt32 8 | { 9 | public: 10 | UInt32 value; 11 | 12 | explicit WrappingUInt32(UInt32 _value) 13 | : value(_value) 14 | {} 15 | 16 | bool operator<(const WrappingUInt32 & other) const 17 | { 18 | return value != other.value && *this <= other; 19 | } 20 | 21 | bool operator<=(const WrappingUInt32 & other) const 22 | { 23 | const UInt32 HALF = static_cast(1) << 31; 24 | return (value <= other.value && other.value - value < HALF) 25 | || (value > other.value && value - other.value > HALF); 26 | } 27 | 28 | bool operator==(const WrappingUInt32 & other) const 29 | { 30 | return value == other.value; 31 | } 32 | }; 33 | 34 | /** Conforming Zxid definition. 35 | * cf. https://github.com/apache/zookeeper/blob/631d1b284f0edb1c4f6b0fb221bf2428aec71aaa/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md#guarantees-properties-and-definitions 36 | * 37 | * But it is better to read this: https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html 38 | * 39 | * Actually here is the definition of Zxid. 40 | * Every change to the ZooKeeper state receives a stamp in the form of a zxid (ZooKeeper Transaction Id). 41 | * This exposes the total ordering of all changes to ZooKeeper. Each change will have a unique zxid 42 | * and if zxid1 is smaller than zxid2 then zxid1 happened before zxid2. 43 | */ 44 | class Zxid 45 | { 46 | public: 47 | WrappingUInt32 epoch; 48 | WrappingUInt32 counter; 49 | explicit Zxid(UInt64 _zxid) 50 | : epoch(static_cast(_zxid >> 32)) 51 | , counter(static_cast(_zxid)) 52 | {} 53 | 54 | bool operator<=(const Zxid & other) const 55 | { 56 | return (epoch < other.epoch) 57 | || (epoch == other.epoch && counter <= other.counter); 58 | } 59 | 60 | bool operator==(const Zxid & other) const 61 | { 62 | return epoch == other.epoch && counter == other.counter; 63 | } 64 | }; 65 | 66 | /* When multiple ClusterCopiers discover that the target partition is not empty, 67 | * they will attempt to clean up this partition before proceeding to copying. 68 | * 69 | * Instead of purging is_dirty, the history of cleaning work is preserved and partition hygiene is established 70 | * based on a happens-before relation between the events. 71 | * This relation is encoded by LogicalClock based on the mzxid of the is_dirty ZNode and is_dirty/cleaned. 72 | * The fact of the partition hygiene is encoded by CleanStateClock. 73 | * 74 | * For you to know what mzxid means: 75 | * 76 | * ZooKeeper Stat Structure: 77 | * The Stat structure for each znode in ZooKeeper is made up of the following fields: 78 | * 79 | * -- czxid 80 | * The zxid of the change that caused this znode to be created. 81 | * 82 | * -- mzxid 83 | * The zxid of the change that last modified this znode. 84 | * 85 | * -- ctime 86 | * The time in milliseconds from epoch when this znode was created. 87 | * 88 | * -- mtime 89 | * The time in milliseconds from epoch when this znode was last modified. 90 | * 91 | * -- version 92 | * The number of changes to the data of this znode. 93 | * 94 | * -- cversion 95 | * The number of changes to the children of this znode. 96 | * 97 | * -- aversion 98 | * The number of changes to the ACL of this znode. 99 | * 100 | * -- ephemeralOwner 101 | * The session id of the owner of this znode if the znode is an ephemeral node. 102 | * If it is not an ephemeral node, it will be zero. 103 | * 104 | * -- dataLength 105 | * The length of the data field of this znode. 106 | * 107 | * -- numChildren 108 | * The number of children of this znode. 109 | * */ 110 | 111 | class LogicalClock 112 | { 113 | public: 114 | std::optional zxid; 115 | 116 | LogicalClock() = default; 117 | 118 | explicit LogicalClock(UInt64 _zxid) 119 | : zxid(_zxid) 120 | {} 121 | 122 | bool hasHappened() const 123 | { 124 | return bool(zxid); 125 | } 126 | 127 | /// happens-before relation with a reasonable time bound 128 | bool happensBefore(const LogicalClock & other) const 129 | { 130 | return !zxid 131 | || (other.zxid && *zxid <= *other.zxid); 132 | } 133 | 134 | bool operator<=(const LogicalClock & other) const 135 | { 136 | return happensBefore(other); 137 | } 138 | 139 | /// strict equality check 140 | bool operator==(const LogicalClock & other) const 141 | { 142 | return zxid == other.zxid; 143 | } 144 | }; 145 | 146 | 147 | class CleanStateClock 148 | { 149 | public: 150 | LogicalClock discovery_zxid; 151 | std::optional discovery_version; 152 | 153 | LogicalClock clean_state_zxid; 154 | std::optional clean_state_version; 155 | 156 | std::shared_ptr stale; 157 | 158 | bool is_clean() const 159 | { 160 | return !is_stale() 161 | && (!discovery_zxid.hasHappened() || (clean_state_zxid.hasHappened() && discovery_zxid <= clean_state_zxid)); 162 | } 163 | 164 | bool is_stale() const 165 | { 166 | return stale->load(); 167 | } 168 | 169 | CleanStateClock( 170 | const zkutil::ZooKeeperPtr & zookeeper, 171 | const String & discovery_path, 172 | const String & clean_state_path) 173 | : stale(std::make_shared(false)) 174 | { 175 | Coordination::Stat stat{}; 176 | String _some_data; 177 | auto watch_callback = 178 | [my_stale = stale] (const Coordination::WatchResponse & rsp) 179 | { 180 | auto logger = getLogger("ClusterCopier"); 181 | if (rsp.error == Coordination::Error::ZOK) 182 | { 183 | switch (rsp.type) /// NOLINT(bugprone-switch-missing-default-case) 184 | { 185 | case Coordination::CREATED: 186 | LOG_DEBUG(logger, "CleanStateClock change: CREATED, at {}", rsp.path); 187 | my_stale->store(true); 188 | break; 189 | case Coordination::CHANGED: 190 | LOG_DEBUG(logger, "CleanStateClock change: CHANGED, at {}", rsp.path); 191 | my_stale->store(true); 192 | } 193 | } 194 | }; 195 | if (zookeeper->tryGetWatch(discovery_path, _some_data, &stat, watch_callback)) 196 | { 197 | discovery_zxid = LogicalClock(stat.mzxid); 198 | discovery_version = stat.version; 199 | } 200 | if (zookeeper->tryGetWatch(clean_state_path, _some_data, &stat, watch_callback)) 201 | { 202 | clean_state_zxid = LogicalClock(stat.mzxid); 203 | clean_state_version = stat.version; 204 | } 205 | } 206 | 207 | bool operator==(const CleanStateClock & other) const 208 | { 209 | return !is_stale() 210 | && !other.is_stale() 211 | && discovery_zxid == other.discovery_zxid 212 | && discovery_version == other.discovery_version 213 | && clean_state_zxid == other.clean_state_zxid 214 | && clean_state_version == other.clean_state_version; 215 | } 216 | 217 | bool operator!=(const CleanStateClock & other) const 218 | { 219 | return !(*this == other); 220 | } 221 | }; 222 | -------------------------------------------------------------------------------- /programs/copier/clickhouse-copier.cpp: -------------------------------------------------------------------------------- 1 | int mainEntryClickHouseClusterCopier(int argc, char ** argv); 2 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ClickHouse/copier/7413a940a7edd48806d5b7c1bd7a32066c1e9090/tests/integration/test_cluster_copier/__init__.py -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs/conf.d/clusters.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | true 6 | 7 | s0_0_0 8 | 9000 9 | 10 | 11 | s0_0_1 12 | 9000 13 | 14 | 15 | 16 | true 17 | 18 | s0_1_0 19 | 9000 20 | 21 | 22 | 23 | 24 | 25 | true 26 | 27 | s1_0_0 28 | 9000 29 | 30 | 31 | s1_0_1 32 | 9000 33 | 34 | 35 | 36 | true 37 | 38 | s1_1_0 39 | 9000 40 | 41 | 42 | 43 | 44 | 45 | true 46 | 47 | s0_0_0 48 | 9000 49 | 50 | 51 | s0_0_1 52 | 9000 53 | 54 | 55 | 56 | 57 | 58 | 59 | s0_0_0 60 | 9000 61 | 62 | 63 | 64 | 65 | 66 | 67 | s1_0_0 68 | 9000 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs/conf.d/clusters_trivial.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | first_trivial 7 | 9000 8 | 9 | 10 | 11 | 12 | 13 | 14 | second_trivial 15 | 9000 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs/conf.d/ddl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | /clickhouse/task_queue/ddl 4 | 5 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs/conf.d/query_log.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | system 9 |
query_log
10 | 11 | 12 | 1000 13 | 14 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs/config-copier.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | information 4 | /var/log/clickhouse-server/copier/log.log 5 | /var/log/clickhouse-server/copier/log.err.log 6 | 1000M 7 | 10 8 | /var/log/clickhouse-server/copier/stderr.log 9 | /var/log/clickhouse-server/copier/stdout.log 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 1 5 | 6 | 5 7 | 1 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | ::/0 16 | 17 | default 18 | default 19 | 20 | 21 | 12345678 22 | 23 | ::/0 24 | 25 | default 26 | default 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs_three_nodes/conf.d/clusters.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | false 6 | 7 | first 8 | 9000 9 | 10 | 11 | 12 | false 13 | 14 | second 15 | 9000 16 | 17 | 18 | 19 | false 20 | 21 | third 22 | 9000 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs_three_nodes/conf.d/ddl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | /clickhouse/task_queue/ddl 4 | 5 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs_three_nodes/config-copier.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | information 4 | /var/log/clickhouse-server/copier/log.log 5 | /var/log/clickhouse-server/copier/log.err.log 6 | 1000M 7 | 10 8 | /var/log/clickhouse-server/copier/stderr.log 9 | /var/log/clickhouse-server/copier/stdout.log 10 | 11 | 12 | 13 | 14 | zoo1 15 | 2181 16 | 17 | 18 | zoo2 19 | 2181 20 | 21 | 22 | zoo3 23 | 2181 24 | 25 | 2000 26 | 27 | 28 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs_three_nodes/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 1 5 | 1 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | ::/0 14 | 15 | default 16 | default 17 | 18 | 19 | 12345678 20 | 21 | ::/0 22 | 23 | default 24 | default 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs_two_nodes/conf.d/clusters.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | false 6 | 7 | first_of_two 8 | 9000 9 | 10 | 11 | 12 | 13 | 14 | false 15 | 16 | second_of_two 17 | 9000 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs_two_nodes/conf.d/ddl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | /clickhouse/task_queue/ddl 4 | 5 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs_two_nodes/conf.d/storage_configuration.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | /jbod1/ 9 | 10 | 11 | /jbod2/ 12 | 13 | 14 | /external/ 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | external 23 | 24 |
25 | jbod1 26 | jbod2 27 |
28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs_two_nodes/config-copier.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | information 4 | /var/log/clickhouse-server/copier/log.log 5 | /var/log/clickhouse-server/copier/log.err.log 6 | 1000M 7 | 10 8 | /var/log/clickhouse-server/copier/stderr.log 9 | /var/log/clickhouse-server/copier/stdout.log 10 | 11 | 12 | 13 | 14 | zoo1 15 | 2181 16 | 17 | 2000 18 | 19 | 20 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/configs_two_nodes/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 1 5 | 1 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | ::/0 14 | 15 | default 16 | default 17 | 18 | 19 | 12345678 20 | 21 | ::/0 22 | 23 | default 24 | default 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task0_description.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 3 4 | 5 | 6 | 7 | 1 8 | 9 | 10 | 11 | 12 | 0 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | cluster0 23 | default 24 | hits 25 | 26 | cluster1 27 | default 28 | hits 29 | 30 | 2 31 | 32 | 3 4 5 6 1 2 0 33 | 34 | 35 | ENGINE=ReplicatedMergeTree PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 36 | 37 | 38 | d + 1 39 | 40 | 41 | d - d = 0 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | true 50 | 51 | s0_0_0 52 | 9000 53 | 54 | 55 | s0_0_1 56 | 9000 57 | 58 | 59 | 60 | true 61 | 62 | s0_1_0 63 | 9000 64 | 65 | 66 | 67 | 68 | 69 | 70 | true 71 | 72 | s1_0_0 73 | 9000 74 | 75 | 76 | s1_0_1 77 | 9000 78 | 79 | 80 | 81 | true 82 | 83 | s1_1_0 84 | 9000 85 | 86 | 87 | 88 | 255.255.255.255 89 | 9000 90 | 91 | 92 | 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_drop_target_partition.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | false 6 | 7 | first_of_two 8 | 9000 9 | 10 | 11 | 12 | 13 | 14 | false 15 | 16 | second_of_two 17 | 9000 18 | 19 | 20 | 21 | 22 | 23 | 2 24 | 25 | 26 | 27 | source 28 | db_drop_target_partition 29 | source 30 | 31 | destination 32 | db_drop_target_partition 33 | destination 34 | 35 | true 36 | 37 | ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) ORDER BY (Column3, Column2, Column1) 38 | rand() 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_month_to_week_description.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 4 | 5 | 6 | 7 | 1 8 | 2 9 | 10 | 11 | 12 | 0 13 | 14 | 15 | 16 | 17 | 18 | cluster0 19 | default 20 | a 21 | 22 | cluster1 23 | default 24 | b 25 | 26 | 31 | 32 | 2 33 | 34 | 35 | ENGINE= 36 | ReplicatedMergeTree 37 | PARTITION BY toMonday(date) 38 | ORDER BY d 39 | 40 | 41 | 42 | jumpConsistentHash(intHash64(d), 2) 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | true 54 | 55 | s0_0_0 56 | 9000 57 | 58 | 59 | s0_0_1 60 | 9000 61 | 62 | 63 | 64 | true 65 | 66 | s0_1_0 67 | 9000 68 | 69 | 70 | 71 | 72 | 73 | 74 | true 75 | 76 | s1_0_0 77 | 9000 78 | 79 | 80 | s1_0_1 81 | 9000 82 | 83 | 84 | 85 | true 86 | 87 | s1_1_0 88 | 9000 89 | 90 | 91 | 92 | 255.255.255.255 93 | 9000 94 | 95 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_no_arg.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 1 6 | 7 | s0_0_0 8 | 9000 9 | 10 | 11 | 12 | 13 | 14 | 15 | 1 16 | 17 | s1_1_0 18 | 9000 19 | 20 | 21 | 22 | 23 | 24 | 1 25 | 26 | 27 | 28 | source_cluster 29 | default 30 | copier_test1 31 | 32 | default_cluster 33 | default 34 | copier_test1_1 35 | ENGINE = MergeTree PARTITION BY date ORDER BY (date, sipHash64(date)) SAMPLE BY sipHash64(date) 36 | rand() 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_no_index.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | false 6 | 7 | s0_0_0 8 | 9000 9 | 10 | 11 | 12 | 13 | 14 | 15 | false 16 | 17 | s1_1_0 18 | 9000 19 | 20 | 21 | 22 | 23 | 24 | 25 | 2 26 | 27 | 28 | 29 | 1 30 | 31 | 32 | 33 | 34 | 0 35 | 36 | 37 | 39 | 40 | 3 41 | 42 | 1 43 | 44 | 45 | 49 | 50 | 51 | 52 | 53 | source_cluster 54 | default 55 | ontime 56 | 57 | 58 | 59 | destination_cluster 60 | default 61 | ontime22 62 | 63 | 64 | 73 | 74 | 75 | 76 | ENGINE = MergeTree() PARTITION BY Year ORDER BY (Year, FlightDate) SETTINGS index_granularity=8192 77 | 78 | 79 | 80 | 81 | jumpConsistentHash(intHash64(Year), 2) 82 | 83 | 84 | 85 | 86 | 98 | 99 | 2017 100 | 101 | 102 | 103 | 104 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_non_partitioned_table.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 1 6 | 7 | s0_0_0 8 | 9000 9 | 10 | 11 | 12 | 13 | 14 | 15 | 1 16 | 17 | s1_1_0 18 | 9000 19 | 20 | 21 | 22 | 23 | 24 | 1 25 | 26 | 27 | 28 | source_cluster 29 | default 30 | copier_test1 31 | 32 | default_cluster 33 | default 34 | copier_test1_1 35 | ENGINE = MergeTree ORDER BY date SETTINGS index_granularity = 8192 36 | rand() 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_self_copy.xml: -------------------------------------------------------------------------------- 1 | 2 | 9440 3 | 4 | 5 | 6 | false 7 | 8 | s0_0_0 9 | 9000 10 | dbuser 11 | 12345678 12 | 0 13 | 14 | 15 | 16 | 17 | 18 | 19 | false 20 | 21 | s0_0_0 22 | 9000 23 | dbuser 24 | 12345678 25 | 0 26 | 27 | 28 | 29 | 30 | 31 | 2 32 | 33 | 34 | 1 35 | 36 | 37 | 38 | 0 39 | 40 | 41 | 42 | 3 43 | 1 44 | 45 | 46 | 47 | 48 | source_cluster 49 | db1 50 | source_table 51 | 52 | destination_cluster 53 | db2 54 | destination_table 55 | 56 | 57 | ENGINE = MergeTree PARTITION BY a ORDER BY a SETTINGS index_granularity = 8192 58 | 59 | 60 | rand() 61 | 62 | 63 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_skip_index.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | false 6 | 7 | first_of_two 8 | 9000 9 | 10 | 11 | 12 | 13 | 14 | false 15 | 16 | second_of_two 17 | 9000 18 | 19 | 20 | 21 | 22 | 23 | 2 24 | 25 | 26 | 27 | source 28 | db_skip_index 29 | source 30 | 31 | destination 32 | db_skip_index 33 | destination 34 | 35 | ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) ORDER BY (Column3, Column2, Column1) 36 | rand() 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_taxi_data.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | false 6 | 7 | first 8 | 9000 9 | 10 | 11 | 12 | false 13 | 14 | second 15 | 9000 16 | 17 | 18 | 19 | false 20 | 21 | third 22 | 9000 23 | 24 | 25 | 26 | 27 | 28 | 2 29 | 30 | 31 | 32 | events 33 | dailyhistory 34 | yellow_tripdata_staging 35 | events 36 | monthlyhistory 37 | yellow_tripdata_staging 38 | Engine=ReplacingMergeTree() PRIMARY KEY (tpep_pickup_datetime, id) ORDER BY (tpep_pickup_datetime, id) PARTITION BY (pickup_location_id, toYYYYMM(tpep_pickup_datetime)) 39 | sipHash64(id) % 3 40 | 41 | 42 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_test_block_size.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 1 4 | 5 | 6 | 7 | 1 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | shard_0_0 17 | default 18 | test_block_size 19 | 20 | cluster1 21 | default 22 | test_block_size 23 | 24 | 25 | '1970-01-01' 26 | 27 | 28 | 29 | ENGINE= 30 | ReplicatedMergeTree 31 | ORDER BY d PARTITION BY partition 32 | 33 | 34 | 35 | jumpConsistentHash(intHash64(d), 2) 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | true 47 | 48 | s0_0_0 49 | 9000 50 | 51 | 52 | s0_0_1 53 | 9000 54 | 55 | 56 | 57 | true 58 | 59 | s0_1_0 60 | 9000 61 | 62 | 63 | 64 | 65 | 66 | 67 | true 68 | 69 | s1_0_0 70 | 9000 71 | 72 | 73 | s1_0_1 74 | 9000 75 | 76 | 77 | 78 | true 79 | 80 | s1_1_0 81 | 9000 82 | 83 | 84 | 85 | 86 | 87 | 88 | true 89 | 90 | s0_0_0 91 | 9000 92 | 93 | 94 | s0_0_1 95 | 9000 96 | 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_trivial.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 3 4 | 5 | 6 | 7 | 1 8 | 9 | 10 | 11 | 12 | 0 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | source_trivial_cluster 23 | default 24 | trivial 25 | 26 | destination_trivial_cluster 27 | default 28 | trivial 29 | 30 | 31 | ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/hits', '{replica}') PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 32 | 33 | 34 | d + 1 35 | 36 | 37 | d - d = 0 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | first_trivial 47 | 9000 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | second_trivial 57 | 9000 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_trivial_without_arguments.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 3 4 | 5 | 6 | 7 | 1 8 | 9 | 10 | 11 | 12 | 0 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | source_trivial_cluster 23 | default 24 | trivial_without_arguments 25 | 26 | destination_trivial_cluster 27 | default 28 | trivial_without_arguments 29 | 30 | 31 | ENGINE=ReplicatedMergeTree() PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 32 | 33 | 34 | d + 1 35 | 36 | 37 | d - d = 0 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | first_trivial 47 | 9000 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | second_trivial 57 | 9000 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_ttl_columns.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | false 6 | 7 | first_of_two 8 | 9000 9 | 10 | 11 | 12 | 13 | 14 | false 15 | 16 | second_of_two 17 | 9000 18 | 19 | 20 | 21 | 22 | 23 | 2 24 | 25 | 26 | 27 | source 28 | db_ttl_columns 29 | source 30 | 31 | destination 32 | db_ttl_columns 33 | destination 34 | 35 | ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) ORDER BY (Column3, Column2, Column1) 36 | rand() 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_ttl_move_to_volume.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | false 6 | 7 | first_of_two 8 | 9000 9 | 10 | 11 | 12 | 13 | 14 | false 15 | 16 | second_of_two 17 | 9000 18 | 19 | 20 | 21 | 22 | 23 | 2 24 | 25 | 26 | 27 | source 28 | db_move_to_volume 29 | source 30 | 31 | destination 32 | db_move_to_volume 33 | destination 34 | 35 | ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) ORDER BY (Column3, Column2, Column1) TTL Column3 + INTERVAL 1 MONTH TO VOLUME 'external' SETTINGS storage_policy = 'external_with_jbods' 36 | rand() 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/task_with_different_schema.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | false 6 | 7 | first_of_two 8 | 9000 9 | 10 | 11 | 12 | 13 | 14 | false 15 | 16 | second_of_two 17 | 9000 18 | 19 | 20 | 21 | 22 | 23 | 2 24 | 25 | 26 | 27 | source 28 | db_different_schema 29 | source 30 | 31 | destination 32 | db_different_schema 33 | destination 34 | 35 | ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) ORDER BY (Column9, Column1, Column2, Column3, Column4) 36 | rand() 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/test_three_nodes.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import logging 5 | import pytest 6 | 7 | from helpers.cluster import ClickHouseCluster 8 | from helpers.test_tools import TSV 9 | 10 | import docker 11 | 12 | CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) 13 | sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) 14 | 15 | cluster = ClickHouseCluster(__file__) 16 | 17 | 18 | @pytest.fixture(scope="module") 19 | def started_cluster(): 20 | global cluster 21 | try: 22 | for name in ["first", "second", "third"]: 23 | cluster.add_instance( 24 | name, 25 | main_configs=[ 26 | "configs_three_nodes/conf.d/clusters.xml", 27 | "configs_three_nodes/conf.d/ddl.xml", 28 | ], 29 | user_configs=["configs_three_nodes/users.xml"], 30 | with_zookeeper=True, 31 | ) 32 | 33 | cluster.start() 34 | yield cluster 35 | 36 | finally: 37 | cluster.shutdown() 38 | 39 | 40 | class Task: 41 | def __init__(self, cluster): 42 | self.cluster = cluster 43 | self.zk_task_path = "/clickhouse-copier/task" 44 | self.container_task_file = "/task_taxi_data.xml" 45 | 46 | for instance_name, _ in cluster.instances.items(): 47 | instance = cluster.instances[instance_name] 48 | instance.copy_file_to_container( 49 | os.path.join(CURRENT_TEST_DIR, "./task_taxi_data.xml"), 50 | self.container_task_file, 51 | ) 52 | logging.debug( 53 | f"Copied task file to container of '{instance_name}' instance. Path {self.container_task_file}" 54 | ) 55 | 56 | def start(self): 57 | for name in ["first", "second", "third"]: 58 | node = cluster.instances[name] 59 | node.query("DROP DATABASE IF EXISTS dailyhistory SYNC;") 60 | node.query("DROP DATABASE IF EXISTS monthlyhistory SYNC;") 61 | 62 | first = cluster.instances["first"] 63 | 64 | # daily partition database 65 | first.query("CREATE DATABASE IF NOT EXISTS dailyhistory on cluster events;") 66 | first.query( 67 | """CREATE TABLE dailyhistory.yellow_tripdata_staging ON CLUSTER events 68 | ( 69 | id UUID DEFAULT generateUUIDv4(), 70 | vendor_id String, 71 | tpep_pickup_datetime DateTime('UTC'), 72 | tpep_dropoff_datetime DateTime('UTC'), 73 | passenger_count Nullable(Float64), 74 | trip_distance String, 75 | pickup_longitude Float64, 76 | pickup_latitude Float64, 77 | rate_code_id String, 78 | store_and_fwd_flag String, 79 | dropoff_longitude Float64, 80 | dropoff_latitude Float64, 81 | payment_type String, 82 | fare_amount String, 83 | extra String, 84 | mta_tax String, 85 | tip_amount String, 86 | tolls_amount String, 87 | improvement_surcharge String, 88 | total_amount String, 89 | pickup_location_id String, 90 | dropoff_location_id String, 91 | congestion_surcharge String, 92 | junk1 String, junk2 String 93 | ) 94 | Engine = ReplacingMergeTree() 95 | PRIMARY KEY (tpep_pickup_datetime, id) 96 | ORDER BY (tpep_pickup_datetime, id) 97 | PARTITION BY (toYYYYMMDD(tpep_pickup_datetime))""" 98 | ) 99 | 100 | first.query( 101 | """CREATE TABLE dailyhistory.yellow_tripdata 102 | ON CLUSTER events 103 | AS dailyhistory.yellow_tripdata_staging 104 | ENGINE = Distributed('events', 'dailyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""" 105 | ) 106 | 107 | first.query( 108 | """INSERT INTO dailyhistory.yellow_tripdata 109 | SELECT * FROM generateRandom( 110 | 'id UUID DEFAULT generateUUIDv4(), 111 | vendor_id String, 112 | tpep_pickup_datetime DateTime(\\'UTC\\'), 113 | tpep_dropoff_datetime DateTime(\\'UTC\\'), 114 | passenger_count Nullable(Float64), 115 | trip_distance String, 116 | pickup_longitude Float64, 117 | pickup_latitude Float64, 118 | rate_code_id String, 119 | store_and_fwd_flag String, 120 | dropoff_longitude Float64, 121 | dropoff_latitude Float64, 122 | payment_type String, 123 | fare_amount String, 124 | extra String, 125 | mta_tax String, 126 | tip_amount String, 127 | tolls_amount String, 128 | improvement_surcharge String, 129 | total_amount String, 130 | pickup_location_id String, 131 | dropoff_location_id String, 132 | congestion_surcharge String, 133 | junk1 String, 134 | junk2 String', 135 | 1, 10, 2) LIMIT 50;""" 136 | ) 137 | 138 | # monthly partition database 139 | first.query("create database IF NOT EXISTS monthlyhistory on cluster events;") 140 | first.query( 141 | """CREATE TABLE monthlyhistory.yellow_tripdata_staging ON CLUSTER events 142 | ( 143 | id UUID DEFAULT generateUUIDv4(), 144 | vendor_id String, 145 | tpep_pickup_datetime DateTime('UTC'), 146 | tpep_dropoff_datetime DateTime('UTC'), 147 | passenger_count Nullable(Float64), 148 | trip_distance String, 149 | pickup_longitude Float64, 150 | pickup_latitude Float64, 151 | rate_code_id String, 152 | store_and_fwd_flag String, 153 | dropoff_longitude Float64, 154 | dropoff_latitude Float64, 155 | payment_type String, 156 | fare_amount String, 157 | extra String, 158 | mta_tax String, 159 | tip_amount String, 160 | tolls_amount String, 161 | improvement_surcharge String, 162 | total_amount String, 163 | pickup_location_id String, 164 | dropoff_location_id String, 165 | congestion_surcharge String, 166 | junk1 String, 167 | junk2 String 168 | ) 169 | Engine = ReplacingMergeTree() 170 | PRIMARY KEY (tpep_pickup_datetime, id) 171 | ORDER BY (tpep_pickup_datetime, id) 172 | PARTITION BY (pickup_location_id, toYYYYMM(tpep_pickup_datetime))""" 173 | ) 174 | 175 | first.query( 176 | """CREATE TABLE monthlyhistory.yellow_tripdata 177 | ON CLUSTER events 178 | AS monthlyhistory.yellow_tripdata_staging 179 | ENGINE = Distributed('events', 'monthlyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""" 180 | ) 181 | 182 | def check(self): 183 | first = cluster.instances["first"] 184 | a = TSV(first.query("SELECT count() from dailyhistory.yellow_tripdata")) 185 | b = TSV(first.query("SELECT count() from monthlyhistory.yellow_tripdata")) 186 | assert a == b, "Distributed tables" 187 | 188 | for instance_name, instance in cluster.instances.items(): 189 | instance = cluster.instances[instance_name] 190 | a = instance.query( 191 | "SELECT count() from dailyhistory.yellow_tripdata_staging" 192 | ) 193 | b = instance.query( 194 | "SELECT count() from monthlyhistory.yellow_tripdata_staging" 195 | ) 196 | assert a == b, "MergeTree tables on each shard" 197 | 198 | a = TSV( 199 | instance.query( 200 | "SELECT sipHash64(*) from dailyhistory.yellow_tripdata_staging ORDER BY id" 201 | ) 202 | ) 203 | b = TSV( 204 | instance.query( 205 | "SELECT sipHash64(*) from monthlyhistory.yellow_tripdata_staging ORDER BY id" 206 | ) 207 | ) 208 | 209 | assert a == b, "Data on each shard" 210 | 211 | for name in ["first", "second", "third"]: 212 | node = cluster.instances[name] 213 | node.query("DROP DATABASE IF EXISTS dailyhistory SYNC;") 214 | node.query("DROP DATABASE IF EXISTS monthlyhistory SYNC;") 215 | 216 | 217 | def execute_task(started_cluster, task, cmd_options): 218 | task.start() 219 | 220 | zk = started_cluster.get_kazoo_client("zoo1") 221 | logging.debug("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) 222 | 223 | # Run cluster-copier processes on each node 224 | docker_api = started_cluster.docker_client.api 225 | copiers_exec_ids = [] 226 | 227 | cmd = [ 228 | "/usr/bin/clickhouse", 229 | "copier", 230 | "--config", 231 | "/etc/clickhouse-server/config-copier.xml", 232 | "--task-path", 233 | task.zk_task_path, 234 | "--task-file", 235 | task.container_task_file, 236 | "--task-upload-force", 237 | "true", 238 | "--base-dir", 239 | "/var/log/clickhouse-server/copier", 240 | ] 241 | cmd += cmd_options 242 | 243 | logging.debug(f"execute_task cmd: {cmd}") 244 | 245 | for instance_name in started_cluster.instances.keys(): 246 | instance = started_cluster.instances[instance_name] 247 | container = instance.get_docker_handle() 248 | instance.copy_file_to_container( 249 | os.path.join(CURRENT_TEST_DIR, "configs_three_nodes/config-copier.xml"), 250 | "/etc/clickhouse-server/config-copier.xml", 251 | ) 252 | logging.info("Copied copier config to {}".format(instance.name)) 253 | exec_id = docker_api.exec_create(container.id, cmd, stderr=True) 254 | output = docker_api.exec_start(exec_id).decode("utf8") 255 | logging.info(output) 256 | copiers_exec_ids.append(exec_id) 257 | logging.info( 258 | "Copier for {} ({}) has started".format(instance.name, instance.ip_address) 259 | ) 260 | 261 | # time.sleep(1000) 262 | 263 | # Wait for copiers stopping and check their return codes 264 | for exec_id, instance in zip( 265 | copiers_exec_ids, iter(started_cluster.instances.values()) 266 | ): 267 | while True: 268 | res = docker_api.exec_inspect(exec_id) 269 | if not res["Running"]: 270 | break 271 | time.sleep(1) 272 | 273 | assert res["ExitCode"] == 0, "Instance: {} ({}). Info: {}".format( 274 | instance.name, instance.ip_address, repr(res) 275 | ) 276 | 277 | try: 278 | task.check() 279 | finally: 280 | zk.delete(task.zk_task_path, recursive=True) 281 | 282 | 283 | # Tests 284 | @pytest.mark.timeout(600) 285 | def test(started_cluster): 286 | execute_task(started_cluster, Task(started_cluster), []) 287 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/test_trivial.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import random 5 | import string 6 | 7 | from helpers.cluster import ClickHouseCluster 8 | from helpers.test_tools import TSV 9 | 10 | import kazoo 11 | import pytest 12 | import docker 13 | 14 | 15 | CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) 16 | sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) 17 | 18 | 19 | COPYING_FAIL_PROBABILITY = 0.1 20 | MOVING_FAIL_PROBABILITY = 0.1 21 | 22 | cluster = ClickHouseCluster(__file__) 23 | 24 | 25 | def generateRandomString(count): 26 | return "".join( 27 | random.choice(string.ascii_uppercase + string.digits) for _ in range(count) 28 | ) 29 | 30 | 31 | @pytest.fixture(scope="module") 32 | def started_cluster(): 33 | global cluster 34 | try: 35 | for name in ["first_trivial", "second_trivial"]: 36 | instance = cluster.add_instance( 37 | name, 38 | main_configs=["configs/conf.d/clusters_trivial.xml"], 39 | user_configs=["configs_two_nodes/users.xml"], 40 | macros={ 41 | "cluster": name, 42 | "shard": "the_only_shard", 43 | "replica": "the_only_replica", 44 | }, 45 | with_zookeeper=True, 46 | ) 47 | 48 | cluster.start() 49 | yield cluster 50 | 51 | finally: 52 | cluster.shutdown() 53 | 54 | 55 | class TaskTrivial: 56 | def __init__(self, cluster): 57 | self.cluster = cluster 58 | self.zk_task_path = "/clickhouse-copier/task_trivial" 59 | self.copier_task_config = open( 60 | os.path.join(CURRENT_TEST_DIR, "task_trivial.xml"), "r" 61 | ).read() 62 | 63 | def start(self): 64 | source = cluster.instances["first_trivial"] 65 | destination = cluster.instances["second_trivial"] 66 | 67 | for node in [source, destination]: 68 | node.query("DROP DATABASE IF EXISTS default") 69 | node.query("CREATE DATABASE IF NOT EXISTS default") 70 | 71 | source.query( 72 | "CREATE TABLE trivial (d UInt64, d1 UInt64 MATERIALIZED d+1)" 73 | "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial/{}', '1') " 74 | "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16".format( 75 | generateRandomString(10) 76 | ) 77 | ) 78 | 79 | source.query( 80 | "INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002", 81 | settings={"distributed_foreground_insert": 1}, 82 | ) 83 | 84 | def check(self): 85 | zk = cluster.get_kazoo_client("zoo1") 86 | status_data, _ = zk.get(self.zk_task_path + "/status") 87 | assert ( 88 | status_data 89 | == b'{"hits":{"all_partitions_count":5,"processed_partitions_count":5}}' 90 | ) 91 | 92 | source = cluster.instances["first_trivial"] 93 | destination = cluster.instances["second_trivial"] 94 | 95 | assert TSV(source.query("SELECT count() FROM trivial")) == TSV("1002\n") 96 | assert TSV(destination.query("SELECT count() FROM trivial")) == TSV("1002\n") 97 | 98 | for node in [source, destination]: 99 | node.query("DROP TABLE trivial") 100 | 101 | 102 | class TaskReplicatedWithoutArguments: 103 | def __init__(self, cluster): 104 | self.cluster = cluster 105 | self.zk_task_path = "/clickhouse-copier/task_trivial_without_arguments" 106 | self.copier_task_config = open( 107 | os.path.join(CURRENT_TEST_DIR, "task_trivial_without_arguments.xml"), "r" 108 | ).read() 109 | 110 | def start(self): 111 | source = cluster.instances["first_trivial"] 112 | destination = cluster.instances["second_trivial"] 113 | 114 | for node in [source, destination]: 115 | node.query("DROP DATABASE IF EXISTS default") 116 | node.query("CREATE DATABASE IF NOT EXISTS default") 117 | 118 | source.query( 119 | "CREATE TABLE trivial_without_arguments ON CLUSTER source_trivial_cluster (d UInt64, d1 UInt64 MATERIALIZED d+1) " 120 | "ENGINE=ReplicatedMergeTree() " 121 | "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16" 122 | ) 123 | 124 | source.query( 125 | "INSERT INTO trivial_without_arguments SELECT * FROM system.numbers LIMIT 1002", 126 | settings={"distributed_foreground_insert": 1}, 127 | ) 128 | 129 | def check(self): 130 | zk = cluster.get_kazoo_client("zoo1") 131 | status_data, _ = zk.get(self.zk_task_path + "/status") 132 | assert ( 133 | status_data 134 | == b'{"hits":{"all_partitions_count":5,"processed_partitions_count":5}}' 135 | ) 136 | 137 | source = cluster.instances["first_trivial"] 138 | destination = cluster.instances["second_trivial"] 139 | 140 | assert TSV( 141 | source.query("SELECT count() FROM trivial_without_arguments") 142 | ) == TSV("1002\n") 143 | assert TSV( 144 | destination.query("SELECT count() FROM trivial_without_arguments") 145 | ) == TSV("1002\n") 146 | 147 | for node in [source, destination]: 148 | node.query("DROP TABLE trivial_without_arguments") 149 | 150 | 151 | def execute_task(started_cluster, task, cmd_options): 152 | task.start() 153 | 154 | zk = started_cluster.get_kazoo_client("zoo1") 155 | print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) 156 | 157 | try: 158 | zk.delete("/clickhouse-copier", recursive=True) 159 | except kazoo.exceptions.NoNodeError: 160 | print("No node /clickhouse-copier. It is Ok in first test.") 161 | 162 | zk_task_path = task.zk_task_path 163 | zk.ensure_path(zk_task_path) 164 | zk.create(zk_task_path + "/description", task.copier_task_config.encode()) 165 | 166 | # Run cluster-copier processes on each node 167 | docker_api = started_cluster.docker_client.api 168 | copiers_exec_ids = [] 169 | 170 | cmd = [ 171 | "/usr/bin/clickhouse", 172 | "copier", 173 | "--config", 174 | "/etc/clickhouse-server/config-copier.xml", 175 | "--task-path", 176 | zk_task_path, 177 | "--base-dir", 178 | "/var/log/clickhouse-server/copier", 179 | ] 180 | cmd += cmd_options 181 | 182 | copiers = list(started_cluster.instances.keys()) 183 | 184 | for instance_name in copiers: 185 | instance = started_cluster.instances[instance_name] 186 | container = instance.get_docker_handle() 187 | instance.copy_file_to_container( 188 | os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), 189 | "/etc/clickhouse-server/config-copier.xml", 190 | ) 191 | print("Copied copier config to {}".format(instance.name)) 192 | exec_id = docker_api.exec_create(container.id, cmd, stderr=True) 193 | output = docker_api.exec_start(exec_id).decode("utf8") 194 | print(output) 195 | copiers_exec_ids.append(exec_id) 196 | print( 197 | "Copier for {} ({}) has started".format(instance.name, instance.ip_address) 198 | ) 199 | 200 | # Wait for copiers stopping and check their return codes 201 | for exec_id, instance_name in zip(copiers_exec_ids, copiers): 202 | instance = started_cluster.instances[instance_name] 203 | while True: 204 | res = docker_api.exec_inspect(exec_id) 205 | if not res["Running"]: 206 | break 207 | time.sleep(0.5) 208 | 209 | assert res["ExitCode"] == 0, "Instance: {} ({}). Info: {}".format( 210 | instance.name, instance.ip_address, repr(res) 211 | ) 212 | 213 | try: 214 | task.check() 215 | finally: 216 | zk.delete(zk_task_path, recursive=True) 217 | 218 | 219 | # Tests 220 | 221 | 222 | def test_trivial_copy(started_cluster): 223 | execute_task(started_cluster, TaskTrivial(started_cluster), []) 224 | 225 | 226 | def test_trivial_without_arguments(started_cluster): 227 | execute_task(started_cluster, TaskReplicatedWithoutArguments(started_cluster), []) 228 | -------------------------------------------------------------------------------- /tests/integration/test_cluster_copier/test_two_nodes.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import logging 5 | import pytest 6 | 7 | from helpers.cluster import ClickHouseCluster 8 | from helpers.test_tools import TSV 9 | 10 | import docker 11 | 12 | CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) 13 | sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) 14 | 15 | cluster = ClickHouseCluster(__file__) 16 | 17 | 18 | @pytest.fixture(scope="module") 19 | def started_cluster(): 20 | global cluster 21 | try: 22 | for name in ["first_of_two", "second_of_two"]: 23 | instance = cluster.add_instance( 24 | name, 25 | main_configs=[ 26 | "configs_two_nodes/conf.d/clusters.xml", 27 | "configs_two_nodes/conf.d/ddl.xml", 28 | "configs_two_nodes/conf.d/storage_configuration.xml", 29 | ], 30 | user_configs=["configs_two_nodes/users.xml"], 31 | with_zookeeper=True, 32 | ) 33 | 34 | cluster.start() 35 | 36 | for name in ["first_of_two", "second_of_two"]: 37 | instance = cluster.instances[name] 38 | instance.exec_in_container(["bash", "-c", "mkdir /jbod1"]) 39 | instance.exec_in_container(["bash", "-c", "mkdir /jbod2"]) 40 | instance.exec_in_container(["bash", "-c", "mkdir /external"]) 41 | 42 | yield cluster 43 | 44 | finally: 45 | cluster.shutdown() 46 | 47 | 48 | # Will copy table from `first` node to `second` 49 | class TaskWithDifferentSchema: 50 | def __init__(self, cluster): 51 | self.cluster = cluster 52 | self.zk_task_path = "/clickhouse-copier/task_with_different_schema" 53 | self.container_task_file = "/task_with_different_schema.xml" 54 | 55 | for instance_name, _ in cluster.instances.items(): 56 | instance = cluster.instances[instance_name] 57 | instance.copy_file_to_container( 58 | os.path.join(CURRENT_TEST_DIR, "./task_with_different_schema.xml"), 59 | self.container_task_file, 60 | ) 61 | print( 62 | "Copied task file to container of '{}' instance. Path {}".format( 63 | instance_name, self.container_task_file 64 | ) 65 | ) 66 | 67 | def start(self): 68 | first = cluster.instances["first_of_two"] 69 | second = cluster.instances["second_of_two"] 70 | 71 | first.query("DROP DATABASE IF EXISTS db_different_schema SYNC") 72 | second.query("DROP DATABASE IF EXISTS db_different_schema SYNC") 73 | 74 | first.query("CREATE DATABASE IF NOT EXISTS db_different_schema;") 75 | first.query( 76 | """CREATE TABLE db_different_schema.source 77 | ( 78 | Column1 String, 79 | Column2 UInt32, 80 | Column3 Date, 81 | Column4 DateTime, 82 | Column5 UInt16, 83 | Column6 String, 84 | Column7 String, 85 | Column8 String, 86 | Column9 String, 87 | Column10 String, 88 | Column11 String, 89 | Column12 Decimal(3, 1), 90 | Column13 DateTime, 91 | Column14 UInt16 92 | ) 93 | ENGINE = MergeTree() 94 | PARTITION BY (toYYYYMMDD(Column3), Column3) 95 | PRIMARY KEY (Column1, Column2, Column3, Column4, Column6, Column7, Column8, Column9) 96 | ORDER BY (Column1, Column2, Column3, Column4, Column6, Column7, Column8, Column9) 97 | SETTINGS index_granularity = 8192""" 98 | ) 99 | 100 | first.query( 101 | """INSERT INTO db_different_schema.source SELECT * FROM generateRandom( 102 | 'Column1 String, Column2 UInt32, Column3 Date, Column4 DateTime, Column5 UInt16, 103 | Column6 String, Column7 String, Column8 String, Column9 String, Column10 String, 104 | Column11 String, Column12 Decimal(3, 1), Column13 DateTime, Column14 UInt16', 1, 10, 2) LIMIT 50;""" 105 | ) 106 | 107 | second.query("CREATE DATABASE IF NOT EXISTS db_different_schema;") 108 | second.query( 109 | """CREATE TABLE db_different_schema.destination 110 | ( 111 | Column1 LowCardinality(String) CODEC(LZ4), 112 | Column2 UInt32 CODEC(LZ4), 113 | Column3 Date CODEC(DoubleDelta, LZ4), 114 | Column4 DateTime CODEC(DoubleDelta, LZ4), 115 | Column5 UInt16 CODEC(LZ4), 116 | Column6 LowCardinality(String) CODEC(ZSTD), 117 | Column7 LowCardinality(String) CODEC(ZSTD), 118 | Column8 LowCardinality(String) CODEC(ZSTD), 119 | Column9 LowCardinality(String) CODEC(ZSTD), 120 | Column10 String CODEC(ZSTD(6)), 121 | Column11 LowCardinality(String) CODEC(LZ4), 122 | Column12 Decimal(3,1) CODEC(LZ4), 123 | Column13 DateTime CODEC(DoubleDelta, LZ4), 124 | Column14 UInt16 CODEC(LZ4) 125 | ) ENGINE = MergeTree() 126 | PARTITION BY toYYYYMMDD(Column3) 127 | ORDER BY (Column9, Column1, Column2, Column3, Column4);""" 128 | ) 129 | 130 | print("Preparation completed") 131 | 132 | def check(self): 133 | first = cluster.instances["first_of_two"] 134 | second = cluster.instances["second_of_two"] 135 | 136 | a = first.query("SELECT count() from db_different_schema.source") 137 | b = second.query("SELECT count() from db_different_schema.destination") 138 | assert a == b, "Count" 139 | 140 | a = TSV( 141 | first.query( 142 | """SELECT sipHash64(*) from db_different_schema.source 143 | ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8, Column9, Column10, Column11, Column12, Column13, Column14)""" 144 | ) 145 | ) 146 | b = TSV( 147 | second.query( 148 | """SELECT sipHash64(*) from db_different_schema.destination 149 | ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8, Column9, Column10, Column11, Column12, Column13, Column14)""" 150 | ) 151 | ) 152 | assert a == b, "Data" 153 | 154 | first.query("DROP DATABASE IF EXISTS db_different_schema SYNC") 155 | second.query("DROP DATABASE IF EXISTS db_different_schema SYNC") 156 | 157 | 158 | # Just simple copying, but table schema has TTL on columns 159 | # Also table will have slightly different schema 160 | class TaskTTL: 161 | def __init__(self, cluster): 162 | self.cluster = cluster 163 | self.zk_task_path = "/clickhouse-copier/task_ttl_columns" 164 | self.container_task_file = "/task_ttl_columns.xml" 165 | 166 | for instance_name, _ in cluster.instances.items(): 167 | instance = cluster.instances[instance_name] 168 | instance.copy_file_to_container( 169 | os.path.join(CURRENT_TEST_DIR, "./task_ttl_columns.xml"), 170 | self.container_task_file, 171 | ) 172 | print( 173 | "Copied task file to container of '{}' instance. Path {}".format( 174 | instance_name, self.container_task_file 175 | ) 176 | ) 177 | 178 | def start(self): 179 | first = cluster.instances["first_of_two"] 180 | second = cluster.instances["second_of_two"] 181 | 182 | first.query("DROP DATABASE IF EXISTS db_ttl_columns SYNC") 183 | second.query("DROP DATABASE IF EXISTS db_ttl_columns SYNC") 184 | 185 | first.query("CREATE DATABASE IF NOT EXISTS db_ttl_columns;") 186 | first.query( 187 | """CREATE TABLE db_ttl_columns.source 188 | ( 189 | Column1 String, 190 | Column2 UInt32, 191 | Column3 Date, 192 | Column4 DateTime, 193 | Column5 UInt16, 194 | Column6 String TTL now() + INTERVAL 1 MONTH, 195 | Column7 Decimal(3, 1) TTL now() + INTERVAL 1 MONTH, 196 | Column8 Tuple(Float64, Float64) TTL now() + INTERVAL 1 MONTH 197 | ) 198 | ENGINE = MergeTree() 199 | PARTITION BY (toYYYYMMDD(Column3), Column3) 200 | PRIMARY KEY (Column1, Column2, Column3) 201 | ORDER BY (Column1, Column2, Column3) 202 | SETTINGS index_granularity = 8192""" 203 | ) 204 | 205 | first.query( 206 | """INSERT INTO db_ttl_columns.source SELECT * FROM generateRandom( 207 | 'Column1 String, Column2 UInt32, Column3 Date, Column4 DateTime, Column5 UInt16, 208 | Column6 String, Column7 Decimal(3, 1), Column8 Tuple(Float64, Float64)', 1, 10, 2) LIMIT 50;""" 209 | ) 210 | 211 | second.query("CREATE DATABASE IF NOT EXISTS db_ttl_columns;") 212 | second.query( 213 | """CREATE TABLE db_ttl_columns.destination 214 | ( 215 | Column1 String, 216 | Column2 UInt32, 217 | Column3 Date, 218 | Column4 DateTime TTL now() + INTERVAL 1 MONTH, 219 | Column5 UInt16 TTL now() + INTERVAL 1 MONTH, 220 | Column6 String TTL now() + INTERVAL 1 MONTH, 221 | Column7 Decimal(3, 1) TTL now() + INTERVAL 1 MONTH, 222 | Column8 Tuple(Float64, Float64) 223 | ) ENGINE = MergeTree() 224 | PARTITION BY toYYYYMMDD(Column3) 225 | ORDER BY (Column3, Column2, Column1);""" 226 | ) 227 | 228 | print("Preparation completed") 229 | 230 | def check(self): 231 | first = cluster.instances["first_of_two"] 232 | second = cluster.instances["second_of_two"] 233 | 234 | a = first.query("SELECT count() from db_ttl_columns.source") 235 | b = second.query("SELECT count() from db_ttl_columns.destination") 236 | assert a == b, "Count" 237 | 238 | a = TSV( 239 | first.query( 240 | """SELECT sipHash64(*) from db_ttl_columns.source 241 | ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8)""" 242 | ) 243 | ) 244 | b = TSV( 245 | second.query( 246 | """SELECT sipHash64(*) from db_ttl_columns.destination 247 | ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8)""" 248 | ) 249 | ) 250 | assert a == b, "Data" 251 | 252 | first.query("DROP DATABASE IF EXISTS db_ttl_columns SYNC") 253 | second.query("DROP DATABASE IF EXISTS db_ttl_columns SYNC") 254 | 255 | 256 | class TaskSkipIndex: 257 | def __init__(self, cluster): 258 | self.cluster = cluster 259 | self.zk_task_path = "/clickhouse-copier/task_skip_index" 260 | self.container_task_file = "/task_skip_index.xml" 261 | 262 | for instance_name, _ in cluster.instances.items(): 263 | instance = cluster.instances[instance_name] 264 | instance.copy_file_to_container( 265 | os.path.join(CURRENT_TEST_DIR, "./task_skip_index.xml"), 266 | self.container_task_file, 267 | ) 268 | print( 269 | "Copied task file to container of '{}' instance. Path {}".format( 270 | instance_name, self.container_task_file 271 | ) 272 | ) 273 | 274 | def start(self): 275 | first = cluster.instances["first_of_two"] 276 | second = cluster.instances["second_of_two"] 277 | 278 | first.query("DROP DATABASE IF EXISTS db_skip_index SYNC") 279 | second.query("DROP DATABASE IF EXISTS db_skip_index SYNC") 280 | 281 | first.query("CREATE DATABASE IF NOT EXISTS db_skip_index;") 282 | first.query( 283 | """CREATE TABLE db_skip_index.source 284 | ( 285 | Column1 UInt64, 286 | Column2 Int32, 287 | Column3 Date, 288 | Column4 DateTime, 289 | Column5 String, 290 | INDEX a (Column1 * Column2, Column5) TYPE minmax GRANULARITY 3, 291 | INDEX b (Column1 * length(Column5)) TYPE set(1000) GRANULARITY 4 292 | ) 293 | ENGINE = MergeTree() 294 | PARTITION BY (toYYYYMMDD(Column3), Column3) 295 | PRIMARY KEY (Column1, Column2, Column3) 296 | ORDER BY (Column1, Column2, Column3) 297 | SETTINGS index_granularity = 8192""" 298 | ) 299 | 300 | first.query( 301 | """INSERT INTO db_skip_index.source SELECT * FROM generateRandom( 302 | 'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""" 303 | ) 304 | 305 | second.query("CREATE DATABASE IF NOT EXISTS db_skip_index;") 306 | second.query( 307 | """CREATE TABLE db_skip_index.destination 308 | ( 309 | Column1 UInt64, 310 | Column2 Int32, 311 | Column3 Date, 312 | Column4 DateTime, 313 | Column5 String, 314 | INDEX a (Column1 * Column2, Column5) TYPE minmax GRANULARITY 3, 315 | INDEX b (Column1 * length(Column5)) TYPE set(1000) GRANULARITY 4 316 | ) ENGINE = MergeTree() 317 | PARTITION BY toYYYYMMDD(Column3) 318 | ORDER BY (Column3, Column2, Column1);""" 319 | ) 320 | 321 | print("Preparation completed") 322 | 323 | def check(self): 324 | first = cluster.instances["first_of_two"] 325 | second = cluster.instances["second_of_two"] 326 | 327 | a = first.query("SELECT count() from db_skip_index.source") 328 | b = second.query("SELECT count() from db_skip_index.destination") 329 | assert a == b, "Count" 330 | 331 | a = TSV( 332 | first.query( 333 | """SELECT sipHash64(*) from db_skip_index.source 334 | ORDER BY (Column1, Column2, Column3, Column4, Column5)""" 335 | ) 336 | ) 337 | b = TSV( 338 | second.query( 339 | """SELECT sipHash64(*) from db_skip_index.destination 340 | ORDER BY (Column1, Column2, Column3, Column4, Column5)""" 341 | ) 342 | ) 343 | assert a == b, "Data" 344 | 345 | first.query("DROP DATABASE IF EXISTS db_skip_index SYNC") 346 | second.query("DROP DATABASE IF EXISTS db_skip_index SYNC") 347 | 348 | 349 | class TaskTTLMoveToVolume: 350 | def __init__(self, cluster): 351 | self.cluster = cluster 352 | self.zk_task_path = "/clickhouse-copier/task_ttl_move_to_volume" 353 | self.container_task_file = "/task_ttl_move_to_volume.xml" 354 | 355 | for instance_name, _ in cluster.instances.items(): 356 | instance = cluster.instances[instance_name] 357 | instance.copy_file_to_container( 358 | os.path.join(CURRENT_TEST_DIR, "./task_ttl_move_to_volume.xml"), 359 | self.container_task_file, 360 | ) 361 | print( 362 | "Copied task file to container of '{}' instance. Path {}".format( 363 | instance_name, self.container_task_file 364 | ) 365 | ) 366 | 367 | def start(self): 368 | first = cluster.instances["first_of_two"] 369 | second = cluster.instances["first_of_two"] 370 | 371 | first.query("DROP DATABASE IF EXISTS db_move_to_volume SYNC") 372 | second.query("DROP DATABASE IF EXISTS db_move_to_volume SYNC") 373 | 374 | first.query("CREATE DATABASE IF NOT EXISTS db_move_to_volume;") 375 | first.query( 376 | """CREATE TABLE db_move_to_volume.source 377 | ( 378 | Column1 UInt64, 379 | Column2 Int32, 380 | Column3 Date, 381 | Column4 DateTime, 382 | Column5 String 383 | ) 384 | ENGINE = MergeTree() 385 | PARTITION BY (toYYYYMMDD(Column3), Column3) 386 | PRIMARY KEY (Column1, Column2, Column3) 387 | ORDER BY (Column1, Column2, Column3) 388 | TTL Column3 + INTERVAL 1 MONTH TO VOLUME 'external' 389 | SETTINGS storage_policy = 'external_with_jbods';""" 390 | ) 391 | 392 | first.query( 393 | """INSERT INTO db_move_to_volume.source SELECT * FROM generateRandom( 394 | 'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""" 395 | ) 396 | 397 | second.query("CREATE DATABASE IF NOT EXISTS db_move_to_volume;") 398 | second.query( 399 | """CREATE TABLE db_move_to_volume.destination 400 | ( 401 | Column1 UInt64, 402 | Column2 Int32, 403 | Column3 Date, 404 | Column4 DateTime, 405 | Column5 String 406 | ) ENGINE = MergeTree() 407 | PARTITION BY toYYYYMMDD(Column3) 408 | ORDER BY (Column3, Column2, Column1) 409 | TTL Column3 + INTERVAL 1 MONTH TO VOLUME 'external' 410 | SETTINGS storage_policy = 'external_with_jbods';""" 411 | ) 412 | 413 | print("Preparation completed") 414 | 415 | def check(self): 416 | first = cluster.instances["first_of_two"] 417 | second = cluster.instances["second_of_two"] 418 | 419 | a = first.query("SELECT count() from db_move_to_volume.source") 420 | b = second.query("SELECT count() from db_move_to_volume.destination") 421 | assert a == b, "Count" 422 | 423 | a = TSV( 424 | first.query( 425 | """SELECT sipHash64(*) from db_move_to_volume.source 426 | ORDER BY (Column1, Column2, Column3, Column4, Column5)""" 427 | ) 428 | ) 429 | b = TSV( 430 | second.query( 431 | """SELECT sipHash64(*) from db_move_to_volume.destination 432 | ORDER BY (Column1, Column2, Column3, Column4, Column5)""" 433 | ) 434 | ) 435 | assert a == b, "Data" 436 | 437 | first.query("DROP DATABASE IF EXISTS db_move_to_volume SYNC") 438 | second.query("DROP DATABASE IF EXISTS db_move_to_volume SYNC") 439 | 440 | 441 | class TaskDropTargetPartition: 442 | def __init__(self, cluster): 443 | self.cluster = cluster 444 | self.zk_task_path = "/clickhouse-copier/task_drop_target_partition" 445 | self.container_task_file = "/task_drop_target_partition.xml" 446 | 447 | for instance_name, _ in cluster.instances.items(): 448 | instance = cluster.instances[instance_name] 449 | instance.copy_file_to_container( 450 | os.path.join(CURRENT_TEST_DIR, "./task_drop_target_partition.xml"), 451 | self.container_task_file, 452 | ) 453 | print( 454 | "Copied task file to container of '{}' instance. Path {}".format( 455 | instance_name, self.container_task_file 456 | ) 457 | ) 458 | 459 | def start(self): 460 | first = cluster.instances["first_of_two"] 461 | second = cluster.instances["second_of_two"] 462 | 463 | first.query("DROP DATABASE IF EXISTS db_drop_target_partition SYNC") 464 | second.query("DROP DATABASE IF EXISTS db_drop_target_partition SYNC") 465 | 466 | first.query("CREATE DATABASE IF NOT EXISTS db_drop_target_partition;") 467 | first.query( 468 | """CREATE TABLE db_drop_target_partition.source 469 | ( 470 | Column1 UInt64, 471 | Column2 Int32, 472 | Column3 Date, 473 | Column4 DateTime, 474 | Column5 String 475 | ) 476 | ENGINE = MergeTree() 477 | PARTITION BY (toYYYYMMDD(Column3), Column3) 478 | PRIMARY KEY (Column1, Column2, Column3) 479 | ORDER BY (Column1, Column2, Column3);""" 480 | ) 481 | 482 | first.query( 483 | """INSERT INTO db_drop_target_partition.source SELECT * FROM generateRandom( 484 | 'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""" 485 | ) 486 | 487 | second.query("CREATE DATABASE IF NOT EXISTS db_drop_target_partition;") 488 | second.query( 489 | """CREATE TABLE db_drop_target_partition.destination 490 | ( 491 | Column1 UInt64, 492 | Column2 Int32, 493 | Column3 Date, 494 | Column4 DateTime, 495 | Column5 String 496 | ) ENGINE = MergeTree() 497 | PARTITION BY toYYYYMMDD(Column3) 498 | ORDER BY (Column3, Column2, Column1);""" 499 | ) 500 | 501 | # Insert data in target too. It has to be dropped. 502 | first.query( 503 | """INSERT INTO db_drop_target_partition.destination SELECT * FROM db_drop_target_partition.source;""" 504 | ) 505 | 506 | print("Preparation completed") 507 | 508 | def check(self): 509 | first = cluster.instances["first_of_two"] 510 | second = cluster.instances["second_of_two"] 511 | 512 | a = first.query("SELECT count() from db_drop_target_partition.source") 513 | b = second.query("SELECT count() from db_drop_target_partition.destination") 514 | assert a == b, "Count" 515 | 516 | a = TSV( 517 | first.query( 518 | """SELECT sipHash64(*) from db_drop_target_partition.source 519 | ORDER BY (Column1, Column2, Column3, Column4, Column5)""" 520 | ) 521 | ) 522 | b = TSV( 523 | second.query( 524 | """SELECT sipHash64(*) from db_drop_target_partition.destination 525 | ORDER BY (Column1, Column2, Column3, Column4, Column5)""" 526 | ) 527 | ) 528 | assert a == b, "Data" 529 | 530 | first.query("DROP DATABASE IF EXISTS db_drop_target_partition SYNC") 531 | second.query("DROP DATABASE IF EXISTS db_drop_target_partition SYNC") 532 | 533 | 534 | def execute_task(started_cluster, task, cmd_options): 535 | task.start() 536 | 537 | zk = started_cluster.get_kazoo_client("zoo1") 538 | print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) 539 | 540 | # Run cluster-copier processes on each node 541 | docker_api = started_cluster.docker_client.api 542 | copiers_exec_ids = [] 543 | 544 | cmd = [ 545 | "/usr/bin/clickhouse", 546 | "copier", 547 | "--config", 548 | "/etc/clickhouse-server/config-copier.xml", 549 | "--task-path", 550 | task.zk_task_path, 551 | "--task-file", 552 | task.container_task_file, 553 | "--task-upload-force", 554 | "true", 555 | "--base-dir", 556 | "/var/log/clickhouse-server/copier", 557 | ] 558 | cmd += cmd_options 559 | 560 | print(cmd) 561 | 562 | for instance_name in started_cluster.instances.keys(): 563 | instance = started_cluster.instances[instance_name] 564 | container = instance.get_docker_handle() 565 | instance.copy_file_to_container( 566 | os.path.join(CURRENT_TEST_DIR, "configs_two_nodes/config-copier.xml"), 567 | "/etc/clickhouse-server/config-copier.xml", 568 | ) 569 | logging.info("Copied copier config to {}".format(instance.name)) 570 | exec_id = docker_api.exec_create(container.id, cmd, stderr=True) 571 | output = docker_api.exec_start(exec_id).decode("utf8") 572 | logging.info(output) 573 | copiers_exec_ids.append(exec_id) 574 | logging.info( 575 | "Copier for {} ({}) has started".format(instance.name, instance.ip_address) 576 | ) 577 | 578 | # time.sleep(1000) 579 | 580 | # Wait for copiers stopping and check their return codes 581 | for exec_id, instance in zip( 582 | copiers_exec_ids, iter(started_cluster.instances.values()) 583 | ): 584 | while True: 585 | res = docker_api.exec_inspect(exec_id) 586 | if not res["Running"]: 587 | break 588 | time.sleep(1) 589 | 590 | assert res["ExitCode"] == 0, "Instance: {} ({}). Info: {}".format( 591 | instance.name, instance.ip_address, repr(res) 592 | ) 593 | 594 | try: 595 | task.check() 596 | finally: 597 | zk.delete(task.zk_task_path, recursive=True) 598 | --------------------------------------------------------------------------------