├── .gitignore ├── LICENSE ├── README.md ├── clickhouse_manager ├── __init__.py ├── chconfigmanager.py ├── chmanager.py ├── cliopts.py ├── config.py ├── main.py └── sshcopier.py ├── config ├── clickhouse-client.xml ├── config.xml ├── config.xml.old ├── config.xml.test └── users.xml ├── package_clear_old.sh ├── package_publish.sh ├── package_source_distr.sh ├── package_wheels_distr.sh ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | bin 11 | var 12 | sdist 13 | develop-eggs 14 | .installed.cfg 15 | 16 | # Installer logs 17 | pip-log.txt 18 | 19 | # Unit test / coverage reports 20 | .coverage 21 | .tox 22 | 23 | # Translations 24 | *.mo 25 | 26 | #Mr Developer 27 | .mr.developer.cfg 28 | 29 | # Doc 30 | _build 31 | 32 | # Text Editor Backupfile 33 | *~ 34 | 35 | # Intellij IDE 36 | .idea 37 | *.xml 38 | *.iml 39 | 40 | # Nose 41 | .noseids 42 | 43 | # Pyenv 44 | .python-version 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Altinity 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # clickhouse-cluster-manager -------------------------------------------------------------------------------- /clickhouse_manager/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from .main import Main 5 | 6 | 7 | def main(): 8 | """Entry point for the application script""" 9 | main = Main() 10 | main.start() 11 | 12 | 13 | if __name__ == '__main__': 14 | main() 15 | -------------------------------------------------------------------------------- /clickhouse_manager/chconfigmanager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import os 6 | import tempfile 7 | import lxml 8 | from lxml import etree 9 | 10 | from .sshcopier import SSHCopier 11 | 12 | 13 | class CHConfigManager: 14 | """ClickHouse configuration manager 15 | 16 | :param config string configuration content 17 | """ 18 | 19 | # string - XML configuration content 20 | ch_config = None 21 | 22 | # Config object 23 | config = None 24 | 25 | def __init__(self, ch_config, config): 26 | self.ch_config = ch_config 27 | self.config = config 28 | 29 | @staticmethod 30 | def is_element_comment(element): 31 | """Check whether specified element is an XML comment 32 | 33 | :param element: Element to check 34 | :return: bool 35 | """ 36 | return isinstance(element, lxml.etree._Comment) 37 | 38 | def add_cluster(self, cluster_name): 39 | """Add new cluster to config 40 | :param cluster_name: 41 | :return: 42 | """ 43 | def on_cluster_root(remote_servers_element): 44 | """ 45 | Add new cluster to the root of cluster specification 46 | :param remote_servers_element: 47 | :return: 48 | """ 49 | new_cluster_element = etree.Element(cluster_name) 50 | remote_servers_element.append(new_cluster_element) 51 | 52 | return self.walk_config(on_cluster_root=on_cluster_root) 53 | 54 | def add_shard(self, cluster_name): 55 | """ 56 | Add new shard into cluster named cluster_name 57 | :param cluster_name: 58 | :return: 59 | """ 60 | def on_cluster(cluster_element, cluster_element_index): 61 | if cluster_element.tag != cluster_name: 62 | # this is not our cluster 63 | return 64 | # this is our cluster, add shard 65 | new_shard_element = etree.Element('shard') 66 | cluster_element.append(new_shard_element) 67 | 68 | return self.walk_config(on_cluster=on_cluster) 69 | 70 | def add_replica(self, cluster_name, shard_index, host, port): 71 | """ 72 | Add new replica with host:port into cluster named cluster_name shard with index shard_index 73 | 74 | :param cluster_name: 75 | :param shard_index: 76 | :param host: 77 | :param port: 78 | :return: 79 | """ 80 | def on_shard(cluster_element, cluster_element_index, shard_element, shard_element_index): 81 | if cluster_element.tag != cluster_name: 82 | # this is not our cluster 83 | return 84 | 85 | if shard_element_index != shard_index: 86 | # this is not our shard 87 | return 88 | 89 | # this is our cluster + shard, add replica 90 | new_replica_element = etree.Element('replica') 91 | 92 | new_host_element = etree.Element('host') 93 | new_host_element.text = host 94 | new_port_element = etree.Element('port') 95 | new_port_element.text = port 96 | 97 | new_replica_element.append(new_host_element) 98 | new_replica_element.append(new_port_element) 99 | 100 | # append replica to the shard 101 | shard_element.append(new_replica_element) 102 | 103 | return self.walk_config(on_shard=on_shard) 104 | 105 | def delete_cluster(self, cluster_name): 106 | """ 107 | Delete cluster from clusters specification 108 | 109 | :param cluster_name: 110 | :return: 111 | """ 112 | def on_cluster(cluster_element, cluster_element_index): 113 | if cluster_element.tag != cluster_name: 114 | # this is not our cluster 115 | return 116 | # this is our cluster, remove current cluster from it's parent 117 | cluster_element.getparent().remove(cluster_element) 118 | 119 | return self.walk_config(on_cluster=on_cluster) 120 | 121 | def delete_shard(self, cluster_name, shard_index): 122 | """ 123 | Delete shard with specified index in specified cluster 124 | :param cluster_name: 125 | :param shard_index: 126 | :return: 127 | """ 128 | def on_shard(cluster_element, cluster_element_index, shard_element, shard_element_index): 129 | if cluster_element.tag != cluster_name: 130 | # this is not our cluster 131 | return 132 | 133 | if shard_element_index != shard_index: 134 | # this is not our shard 135 | return 136 | 137 | # this is our cluster and our shard 138 | cluster_element.remove(shard_element) 139 | 140 | return self.walk_config(on_shard=on_shard) 141 | 142 | def delete_replica(self, cluster_name, shard_index, host, port): 143 | """ 144 | Delete replica having host:port inside shard with specified index in specified cluster 145 | :param cluster_name: 146 | :param shard_index: 147 | :param host: 148 | :param port: 149 | :return: 150 | """ 151 | def on_replica(cluster_element, cluster_element_index, shard_element, shard_element_index, replica_element, replica_element_index): 152 | if cluster_element.tag != cluster_name: 153 | # this is not our cluster 154 | return 155 | 156 | if shard_element_index != shard_index: 157 | # this is not our shard 158 | return 159 | 160 | # this is our cluster and our shard 161 | shard_element.remove(replica_element) 162 | 163 | return self.walk_config(on_replica=on_replica) 164 | 165 | def print(self): 166 | """ 167 | Print cluster specification 168 | :return: 169 | """ 170 | def on_cluster(cluster_element, cluster_element_index): 171 | """Callback on_cluster""" 172 | print() 173 | print(cluster_element.tag) 174 | pass 175 | 176 | def on_shard(cluster_element, cluster_element_index, shard_element, shard_element_index): 177 | """Callback on_shard""" 178 | print(' ' + shard_element.tag + '[' + str(shard_element_index) + ']') 179 | pass 180 | 181 | def on_replica(cluster_element, cluster_element_index, shard_element, shard_element_index, replica_element, replica_element_index): 182 | """Callback on_replica""" 183 | host_element = replica_element.find('host') 184 | port_element = replica_element.find('port') 185 | print(" " + replica_element.tag + '[' + str(replica_element_index) + "]|" + host_element.tag + ":" + host_element.text + ":" + port_element.tag + ":" + port_element.text + " path: " + cluster_element.tag + '/' + shard_element.tag + '[' + str(shard_element_index) + ']/' + replica_element.tag) 186 | pass 187 | 188 | return self.walk_config(on_cluster=on_cluster, on_shard=on_shard, on_replica=on_replica) 189 | 190 | def push(self): 191 | """ 192 | Push configuration onto all replicas found in cluster specification 193 | :return: 194 | """ 195 | def on_replica(cluster_element, cluster_element_index, shard_element, shard_element_index, replica_element, replica_element_index): 196 | """ 197 | Callback on_replica 198 | Accumulate all replica specifications 199 | """ 200 | # extract host:port from child tags of 201 | host_element = replica_element.find('host') 202 | port_element = replica_element.find('port') 203 | print(" " + replica_element.tag + '[' + str(replica_element_index) + "]|" + host_element.tag + ":" + host_element.text + ":" + port_element.tag + ":" + port_element.text + " path: " + cluster_element.tag + '/' + shard_element.tag + '[' + str(shard_element_index) + ']/' + replica_element.tag) 204 | host = host_element.text 205 | port = port_element.text 206 | # accumulate {host:HOST, port:9000} dict 207 | on_replica.hosts.append({'host': host, 'port':port}) 208 | 209 | # accumulate all replica specifications 210 | on_replica.hosts = [] 211 | self.walk_config(on_replica=on_replica) 212 | 213 | # save config to temp file 214 | fd, tempfile_path = tempfile.mkstemp() 215 | os.write(fd, self.ch_config) 216 | os.close(fd) 217 | print("Save config as %(tmpfile)s" % {'tmpfile': tempfile_path}) 218 | 219 | # walk over all replica specifications and SSH copy config onto it 220 | for replica in on_replica.hosts: 221 | # where config would be copied to 222 | host = replica['host'] 223 | print("Pushing to:" + host) 224 | 225 | # 226 | # SSH copy config file to replica 227 | # 228 | 229 | # copy temp file 230 | copier = SSHCopier( 231 | hostname=host, 232 | username=self.config.ssh_username(), 233 | password=self.config.ssh_password(), 234 | dir_remote='/etc/clickhouse-server/', 235 | files_to_copy=[tempfile_path], 236 | dry=self.config.dry() 237 | ) 238 | copier.copy_files_list() 239 | 240 | # remove temp file 241 | os.remove(tempfile_path) 242 | 243 | # lxml.etree._Element 244 | # def on_cluster(self, cluster_element): 245 | # print("cluster: " + cluster_element.tag) 246 | # 247 | # def on_shard(self, cluster_element, shard_element): 248 | # print(" shard: " + shard_element.tag + " path: " + cluster_element.tag + '/' + shard_element.tag) 249 | # 250 | # def on_replica(self, cluster_element, shard_element, replica_element): 251 | # host_element = replica_element.find('host') 252 | # port_element = replica_element.find('port') 253 | # print(" replica: " + replica_element.tag + "|" + host_element.tag + ":" + host_element.text + ":" + port_element.tag + ":" + port_element.text + " path: " + cluster_element.tag + '/' + shard_element.tag + '/' + replica_element.tag) 254 | 255 | def walk_config( 256 | self, 257 | on_cluster_root=None, 258 | on_cluster=None, 259 | on_shard=None, 260 | on_replica=None 261 | ): 262 | """ 263 | Walk over cluster configuration calling callback functions on-the-way 264 | 265 | :param on_cluster_root: 266 | :param on_cluster: 267 | :param on_shard: 268 | :param on_replica: 269 | :return: 270 | """ 271 | try: 272 | # ElementTree object 273 | config_tree = etree.fromstring(self.ch_config, etree.XMLParser(remove_blank_text=True, encoding="utf-8")) 274 | except IOError: 275 | # file is not readable 276 | print("IOError") 277 | return 278 | except etree.XMLSyntaxError: 279 | # file is readable, but has does not contain well-formed XML 280 | print("SyntaxError") 281 | return 282 | 283 | 284 | # config_root = config_tree.getroot() 285 | # '' 286 | remote_servers_element = config_tree.find('remote_servers') 287 | 288 | if remote_servers_element is None: 289 | # no tag available 290 | return 291 | 292 | # found 293 | 294 | if callable(on_cluster_root): 295 | on_cluster_root(remote_servers_element) 296 | 297 | # iterate over children elements 298 | # each tag inside it would be name of the cluster. ex: 299 | 300 | if not len(remote_servers_element): 301 | print("No clusters defined") 302 | 303 | cluster_element_index = 0 304 | 305 | # walk over clusters inside 'remote servers' 306 | for cluster_element in remote_servers_element: 307 | 308 | # skip comments 309 | if self.is_element_comment(cluster_element): 310 | continue 311 | 312 | # normal element - cluster name 313 | 314 | if callable(on_cluster): 315 | on_cluster(cluster_element, cluster_element_index) 316 | 317 | # shards have no names, so they need to be indexed in order to be accessed personally 318 | shard_element_index = 0 319 | 320 | # walk over shards inside cluster 321 | for shard_element in cluster_element: 322 | 323 | # skip comments 324 | if self.is_element_comment(shard_element): 325 | continue 326 | # skip everything what is not tag 327 | if shard_element.tag != 'shard': 328 | continue 329 | 330 | # normal element - 331 | if callable(on_shard): 332 | on_shard(cluster_element, cluster_element_index, shard_element, shard_element_index) 333 | 334 | # replicas have no names, so they need to be indexed in order to be accessed personally 335 | replica_element_index = 0 336 | 337 | # walk over replicas inside shard 338 | for replica_element in shard_element: 339 | 340 | # skip comments 341 | if self.is_element_comment(replica_element): 342 | continue 343 | # skip everything what is not tag 344 | if replica_element.tag != 'replica': 345 | continue 346 | 347 | # normal element - 348 | if callable(on_replica): 349 | on_replica(cluster_element, cluster_element_index, shard_element, shard_element_index, replica_element, replica_element_index) 350 | 351 | replica_element_index += 1 352 | 353 | shard_element_index += 1 354 | 355 | cluster_element_index += 1 356 | 357 | # new_host_element = etree.Element('host') 358 | # new_host_element.text = 'super-duper-host' 359 | # new_port_element = etree.Element('port') 360 | # new_port_element.text = '9001' 361 | # 362 | # new_replica_element = etree.Element('replica') 363 | # new_replica_element.append(new_host_element) 364 | # new_replica_element.append(new_port_element) 365 | # 366 | # shard_element.append(new_replica_element) 367 | # 368 | 369 | # buld XML out of elements tree we have 370 | self.ch_config = etree.tostring(config_tree, pretty_print=True) 371 | return self.ch_config 372 | -------------------------------------------------------------------------------- /clickhouse_manager/chmanager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import logging 6 | import pprint 7 | import re 8 | 9 | from .cliopts import CLIOpts 10 | from .chconfigmanager import CHConfigManager 11 | 12 | 13 | class CHManager: 14 | """ 15 | High-level class for managing CH cluster configuration 16 | """ 17 | config = None 18 | ch_config = None 19 | ch_config_manager = None 20 | 21 | def __init__(self): 22 | self.config = CLIOpts.config() 23 | 24 | logging.basicConfig( 25 | filename=self.config.log_file(), 26 | level=self.config.log_level(), 27 | format='%(asctime)s/%(created)f:%(levelname)s:%(message)s' 28 | ) 29 | logging.info('Starting') 30 | logging.debug(pprint.pformat(self.config.config)) 31 | 32 | def open_config(self): 33 | try: 34 | f = open(self.config.ch_config_file(), 'rb') 35 | self.ch_config = f.read() 36 | f.close() 37 | return True 38 | except: 39 | return False 40 | 41 | def write_config(self): 42 | f = open('test.xml', 'wb') 43 | f.write(self.ch_config) 44 | f.close() 45 | 46 | @staticmethod 47 | def cluster_path_parse(line): 48 | """ 49 | Parse cluster-address line specification 50 | :param line: 51 | :return: dict 52 | """ 53 | # /cluster/0/host:port 54 | # /cluster/shard0/host:port 55 | 56 | line = line.strip() 57 | 58 | # ensure starting '/' 59 | if not line.startswith('/'): 60 | line = '/' + line 61 | 62 | try: 63 | parts = line.split('/') 64 | except: 65 | parts = [] 66 | 67 | # parts[0] would be empty, ignore it 68 | # fetch cluster - parts[1] 69 | try: 70 | cluster = parts[1] 71 | except IndexError: 72 | # parts[1] - cluster - unavailable 73 | cluster = None 74 | 75 | # fetch shard - parts[2] 76 | try: 77 | # would like to consume both ways 78 | # .../0/... 79 | # .../shard0/... 80 | # so strip all non-numbers in the list 81 | shard_index = int(re.sub('[^0-9]', '', parts[2])) 82 | except IndexError: 83 | # parts[2] - chard - unavailable or malformed 84 | shard_index = None 85 | 86 | # fetch host:port - parts[3] 87 | try: 88 | host_port = parts[3] 89 | host_port = host_port.split(':') 90 | host = host_port[0] 91 | port = host_port[1] 92 | except IndexError: 93 | # parts[3] - host:port - unavailable or malformed 94 | host = None 95 | port = None 96 | 97 | return { 98 | 'cluster': cluster, 99 | 'shard_index': shard_index, 100 | 'host': host, 101 | 'port': port 102 | } 103 | 104 | @staticmethod 105 | def cluster_path_print(): 106 | print('Cluster path example /cluster1/shard0/host:port') 107 | pass 108 | 109 | def add_cluster(self): 110 | """High-level add cluster""" 111 | print("Add cluster") 112 | c = self.cluster_path_parse(input("Cluster name to add:")) 113 | print(c) 114 | self.ch_config = self.ch_config_manager.add_cluster(c['cluster']) 115 | 116 | def add_shard(self): 117 | """High-level add shard""" 118 | print("Add shard") 119 | c = self.cluster_path_parse(input("Cluster name to add shard:")) 120 | print(c) 121 | self.ch_config = self.ch_config_manager.add_shard(c['cluster']) 122 | 123 | def add_replica(self): 124 | """High-level add replica""" 125 | print("Add replica") 126 | self.cluster_path_print() 127 | c = self.cluster_path_parse(input("Cluster path for replica:")) 128 | print(c) 129 | self.ch_config = self.ch_config_manager.add_replica(c['cluster'], c['shard_index'], c['host'], c['port']) 130 | 131 | def delete_cluster(self): 132 | """High-level delete cluster""" 133 | print("Delete cluster") 134 | c = self.cluster_path_parse(input("Cluster name to delete:")) 135 | print(c) 136 | self.ch_config = self.ch_config_manager.delete_cluster(c['cluster']) 137 | 138 | def delete_shard(self): 139 | """High-level delete shard""" 140 | print("Delete shard") 141 | c = self.cluster_path_parse(input("Cluster path for shard:")) 142 | print(c) 143 | self.ch_config = self.ch_config_manager.delete_shard(c['cluster'], c['shard_index']) 144 | 145 | def delete_replica(self): 146 | """High-level delete replica""" 147 | print("Delete replica") 148 | self.cluster_path_print() 149 | c = self.cluster_path_parse(input("Cluster path for replica:")) 150 | print(c) 151 | self.ch_config = self.ch_config_manager.delete_replica(c['cluster'], c['shard_index'], c['host'], c['port']) 152 | 153 | def print(self): 154 | """High-level print config""" 155 | print("Print cluster layout") 156 | self.ch_config_manager.print() 157 | 158 | def write(self): 159 | """High-level write config""" 160 | print("Write cluster layout to disk") 161 | self.write_config() 162 | 163 | def push(self): 164 | """High-level push config""" 165 | self.ch_config_manager.push() 166 | print("pUsh config everywhere") 167 | 168 | @staticmethod 169 | def get_interactive_choice(): 170 | print() 171 | print("[1] Add cluster") 172 | print("[2] Add shard") 173 | print("[3] Add replica") 174 | print() 175 | print("[a] Delete cluster") 176 | print("[s] Delete shard") 177 | print("[d] Delete replica") 178 | print() 179 | print("[p] Print cluster layout") 180 | print("[w] Write cluster layout") 181 | print("[u] pUsh cluster config") 182 | print() 183 | print("[q] Quit.") 184 | 185 | return input("What would you like to do? ") 186 | 187 | def interactive(self): 188 | choice = '' 189 | while choice != 'q': 190 | choice = self.get_interactive_choice() 191 | 192 | if choice == '1': 193 | self.add_cluster() 194 | elif choice == '2': 195 | self.add_shard() 196 | elif choice == '3': 197 | self.add_replica() 198 | elif choice == 'a': 199 | self.delete_cluster() 200 | elif choice == 's': 201 | self.delete_shard() 202 | elif choice == 'd': 203 | self.delete_replica() 204 | elif choice == 'p': 205 | self.print() 206 | elif choice == 'w': 207 | self.write() 208 | elif choice == 'u': 209 | self.push() 210 | elif choice == 'q': 211 | print("Thanks for playing. Bye.") 212 | else: 213 | print("Can't understand that choice.") 214 | 215 | def main(self): 216 | """Main function. Global entry point.""" 217 | if not self.open_config(): 218 | print("Can't open config file {}".format(self.config.ch_config_file())) 219 | return 220 | 221 | self.ch_config_manager = CHConfigManager(self.ch_config, self.config) 222 | if self.config.interactive(): 223 | self.interactive() 224 | else: 225 | print("Command mode not implemented yet") 226 | -------------------------------------------------------------------------------- /clickhouse_manager/cliopts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import argparse 6 | import logging 7 | import os 8 | 9 | from .config import Config 10 | 11 | 12 | class CLIOpts(object): 13 | 14 | @staticmethod 15 | def join(lists_to_join): 16 | """Join several lists into one 17 | 18 | :param lists_to_join: is a list of lists 19 | [['a=b', 'c=d'], ['e=f', 'z=x'], ] 20 | 21 | :return: None or dictionary 22 | {'a': 'b', 'c': 'd', 'e': 'f', 'z': 'x'} 23 | 24 | """ 25 | 26 | if not isinstance(lists_to_join, list): 27 | return None 28 | 29 | res = {} 30 | for lst in lists_to_join: 31 | # lst = ['a=b', 'c=d'] 32 | for column_value_pair in lst: 33 | # column_value_value = 'a=b' 34 | column, value = column_value_pair.split('=', 2) 35 | res[column] = value 36 | 37 | # res = dict { 38 | # 'col1': 'value1', 39 | # 'col2': 'value2', 40 | # } 41 | 42 | # return with sanity check 43 | if len(res) > 0: 44 | return res 45 | else: 46 | return None 47 | 48 | @staticmethod 49 | def log_level_from_string(log_level_string): 50 | """Convert string representation of a log level into logging.XXX constant""" 51 | 52 | level = log_level_string.upper() 53 | 54 | if level == 'CRITICAL': 55 | return logging.CRITICAL 56 | if level == 'ERROR': 57 | return logging.ERROR 58 | if level == 'WARNING': 59 | return logging.WARNING 60 | if level == 'INFO': 61 | return logging.INFO 62 | if level == 'DEBUG': 63 | return logging.DEBUG 64 | if level == 'NOTSET': 65 | return logging.NOTSET 66 | 67 | return logging.NOTSET 68 | 69 | @staticmethod 70 | def config(): 71 | """Parse application's CLI options into options dictionary 72 | :return: instance of Config 73 | """ 74 | 75 | argparser = argparse.ArgumentParser( 76 | description='ClickHouse configuration manager', 77 | epilog='===============' 78 | ) 79 | argparser.add_argument( 80 | '--interactive', 81 | action='store_true', 82 | help='Interactive mode' 83 | ) 84 | argparser.add_argument( 85 | '--log-file', 86 | type=str, 87 | default=None, 88 | help='Path to log file. Default - not specified' 89 | ) 90 | argparser.add_argument( 91 | '--log-level', 92 | type=str, 93 | default="NOTSET", 94 | help='Log Level. Default - NOTSET' 95 | ) 96 | argparser.add_argument( 97 | '--pid-file', 98 | type=str, 99 | default='/tmp/reader.pid', 100 | help='Pid file to be used by the app' 101 | ) 102 | argparser.add_argument( 103 | '--dry', 104 | action='store_true', 105 | help='Dry mode - do not do anyting that can harm. ' 106 | 'Config files will not be pushed/written/etc. Just simulate. ' 107 | 'Useful for debugging. ' 108 | ) 109 | argparser.add_argument( 110 | '--config-file', 111 | type=str, 112 | default='', 113 | help='Path to CH server config file to work with. Default - not specified' 114 | ) 115 | argparser.add_argument( 116 | '--ssh-user', 117 | type=str, 118 | default='root', 119 | help='username to be used when pushing on servers' 120 | ) 121 | argparser.add_argument( 122 | '--ssh-password', 123 | type=str, 124 | default='', 125 | help='password to be used when pushing on servers' 126 | ) 127 | argparser.add_argument( 128 | '--ssh-port', 129 | type=str, 130 | default='22', 131 | help='port to be used when pushing on servers' 132 | ) 133 | argparser.add_argument( 134 | '--config-folder', 135 | type=str, 136 | default='/etc/clickhouse-server/', 137 | help='Path to CH server config folder. Default value=/etc/clickhouse-server/' 138 | ) 139 | argparser.add_argument( 140 | '--config.xml', 141 | type=str, 142 | default='config.xml', 143 | help='CH server config file. Default value=config.xml' 144 | ) 145 | argparser.add_argument( 146 | '--user.xml', 147 | type=str, 148 | default='user.xml', 149 | help='CH server user file. Default value=user.xml' 150 | ) 151 | 152 | args = argparser.parse_args() 153 | 154 | # build options 155 | return Config({ 156 | 157 | 'app': { 158 | 'interactive': args.interactive, 159 | 'config-file': args.config_file, 160 | 'dry': args.dry, 161 | 'log-file': args.log_file, 162 | 'log-level': CLIOpts.log_level_from_string(args.log_level), 163 | 'pid_file': args.pid_file, 164 | }, 165 | 166 | 'ssh': { 167 | 'username': args.ssh_user, 168 | 'password': args.ssh_password, 169 | 'port': args.ssh_port, 170 | }, 171 | 172 | 'manager': { 173 | 'config-folder': os.path.abspath(args.config_folder), 174 | 'config.xml': os.path.abspath(args.config_folder + '/' + getattr(args, 'config.xml')), 175 | 'user.xml': os.path.abspath(args.config_folder + '/' + getattr(args, 'user.xml')) 176 | }, 177 | }) 178 | -------------------------------------------------------------------------------- /clickhouse_manager/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | class Config(object): 6 | 7 | config = None 8 | 9 | def __init__(self, config): 10 | self.config = config 11 | 12 | def __str__(self): 13 | return str(self.config) 14 | 15 | def __getitem__(self, item): 16 | return self.config[item] 17 | 18 | def interactive(self): 19 | return self.config['app']['interactive'] 20 | 21 | def dry(self): 22 | return self.config['app']['dry'] 23 | 24 | def log_file(self): 25 | return self.config['app']['log-file'] 26 | 27 | def log_level(self): 28 | return self.config['app']['log-level'] 29 | 30 | def pid_file(self): 31 | return self.config['app']['pid_file'] 32 | 33 | def ch_config_folder(self): 34 | return self.config['manager']['config-folder'] 35 | 36 | def ch_config_file(self): 37 | return self.config['manager']['config.xml'] 38 | 39 | def ch_config_user_file(self): 40 | return self.config['manager']['user.xml'] 41 | 42 | def ssh_username(self): 43 | return self.config['ssh']['username'] 44 | 45 | def ssh_password(self): 46 | return self.config['ssh']['password'] 47 | 48 | def ssh_port(self): 49 | return self.config['ssh']['port'] 50 | -------------------------------------------------------------------------------- /clickhouse_manager/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | from .chmanager import CHManager 6 | 7 | 8 | class Main(object): 9 | def start(self): 10 | # print("RUN") 11 | manager = CHManager(); 12 | manager.main() 13 | 14 | 15 | 16 | # from StringIO import StringIO 17 | # from lxml import etree 18 | # from lxml.etree import Element 19 | # 20 | # data = """ 21 | # 22 | # cherry 23 | # apple 24 | # chocolate 25 | # 26 | # """ 27 | 28 | # stream = StringIO(data) 29 | # context = etree.iterparse(stream, events=("start", )) 30 | # 31 | # for action, elem in context: 32 | # if elem.tag == 'items': 33 | # items = elem 34 | # index = 1 35 | # elif elem.tag == 'pie': 36 | # item = Element('item', {'id': str(index)}) 37 | # items.replace(elem, item) 38 | # item.append(elem) 39 | # index += 1 40 | # 41 | # print etree.tostring(context.root) 42 | # 43 | # prints: 44 | # 45 | # 46 | # 47 | # cherry 48 | # apple 49 | # chocolate 50 | # 51 | # 52 | # 53 | # 54 | # 55 | # 56 | # 1 57 | # kites 58 | # kites 59 | # 60 | # 61 | # example = etree.Element("example") 62 | # login = etree.SubElement(example, "login") 63 | # password = etree.SubElement(login,"password") 64 | # password.text = "newPassword" 65 | 66 | if __name__ == '__main__': 67 | main = Main() 68 | main.start() 69 | -------------------------------------------------------------------------------- /clickhouse_manager/sshcopier.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import paramiko 6 | 7 | 8 | class SSHCopier: 9 | """Copy files via SSH 10 | 11 | :param hostname='127.0.0.1', 12 | :param port=22, 13 | :param username='user', 14 | :param password='password', 15 | :param rsa_private_key_filename='~/.ssh/rsa_private_key', 16 | :param dir_remote='/home/to', 17 | :param files_to_copy=[], 18 | :param dry=False 19 | """ 20 | 21 | # remote SSH server hostname 22 | hostname = None 23 | 24 | # remote SSH server port 25 | port = None 26 | 27 | # remote SSH username 28 | username = None 29 | 30 | # remote SSH password 31 | password = None 32 | 33 | # local path to private RSA key, typically ~/.ssh/rsa_key_for_remote_host 34 | rsa_private_key_filename = None 35 | 36 | # dir on remote SSH server where files would be copied 37 | dir_remote = None 38 | 39 | # list of files to copy 40 | files_to_copy = None 41 | 42 | def __init__( 43 | self, 44 | hostname='127.0.0.1', 45 | port=22, 46 | username='user', 47 | password='password', 48 | rsa_private_key_filename='~/.ssh/rsa_private_key', 49 | dir_remote='/home/to', 50 | files_to_copy=[], 51 | dry=False 52 | ): 53 | self.hostname = hostname 54 | self.port = port 55 | self.username = username 56 | self.password = password 57 | self.rsa_private_key_filename = rsa_private_key_filename 58 | self.dir_remote = dir_remote 59 | self.files_to_copy = files_to_copy 60 | self.dry = dry 61 | 62 | def copy_files_list(self): 63 | """Copy list of files 64 | 65 | At first try to connect using a private key either from a private key file 66 | or provided by an SSH agent. 67 | If RSA authentication fails, then make second attempt 68 | with password login. 69 | """ 70 | 71 | # get host key, if we know one 72 | hostkeytype = None 73 | hostkey = None 74 | files_copied = 0 75 | 76 | if self.dry: 77 | # just print what we'd like to do in here 78 | # do actions would be taken 79 | for file in self.files_to_copy: 80 | print("DRY: copy %(file)s to %(hostname)s:%(port)s/%(dir)s as %(username)s:%(password)s" % { 81 | 'file': file, 82 | 'hostname': self.hostname, 83 | 'port': self.port, 84 | 'dir': self.dir_remote, 85 | 'username': self.username, 86 | 'password': '***' 87 | }) 88 | # no actual actions would be taken - nothing to do in this method any more 89 | return 90 | 91 | # build dictionary of known hosts 92 | try: 93 | host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts')) 94 | except: 95 | # can't open known_hosts file, assume it's empty 96 | host_keys = {} 97 | 98 | if self.hostname in host_keys: 99 | # already known host 100 | hostkeytype = host_keys[self.hostname].keys()[0] 101 | hostkey = host_keys[self.hostname][hostkeytype] 102 | print('Using host key of type ' + hostkeytype) 103 | 104 | # connect 105 | try: 106 | print('Establishing SSH connection to:', self.hostname, self.port, '...') 107 | transport = paramiko.Transport((self.hostname, self.port)) 108 | transport.start_client() 109 | except: 110 | # unable to connect at all 111 | return 112 | 113 | # key auth 114 | # try to authenticate with any of: 115 | # 1. private keys available from SSH agent 116 | # 2. local private RSA key file (assumes no pass phrase required) 117 | 118 | # load available keys 119 | rsa_keys = paramiko.Agent().get_keys() 120 | 121 | # append key from provided key file to other available keys 122 | try: 123 | key = paramiko.RSAKey.from_private_key_file(self.rsa_private_key_filename) 124 | rsa_keys += (key,) 125 | except: 126 | print('Failed loading RSA private key', self.rsa_private_key_filename) 127 | 128 | if len(rsa_keys) > 0: 129 | # have RSA keys, try to auth with all of them 130 | for key in rsa_keys: 131 | try: 132 | transport.auth_publickey(self.username, key) 133 | # auth succeeded with this key 134 | # not need to continue with next key 135 | break 136 | except: 137 | # auth failed with this key, continue with next key 138 | pass 139 | 140 | if not transport.is_authenticated(): 141 | # key auth not performed or failed - try username/password 142 | transport.auth_password(username=self.username, password=self.password) 143 | else: 144 | # key auth completed successfully 145 | sftp = transport.open_session() 146 | 147 | sftp = paramiko.SFTPClient.from_transport(transport) 148 | 149 | # create remote dir 150 | try: 151 | sftp.mkdir(self.dir_remote) 152 | except: 153 | # may be remote dir already exists 154 | pass 155 | 156 | # copy files 157 | for filename in self.files_to_copy: 158 | remote_filepath = self.dir_remote + '/' + os.path.basename(filename) 159 | try: 160 | sftp.put(filename, remote_filepath) 161 | files_copied += 1 162 | except: 163 | # file not copied, skip so far 164 | pass 165 | 166 | sftp.close() 167 | transport.close() 168 | 169 | return files_copied 170 | -------------------------------------------------------------------------------- /config/clickhouse-client.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /config/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | trace 5 | /var/log/clickhouse-server/clickhouse-server.log 6 | /var/log/clickhouse-server/clickhouse-server.err.log 7 | 1000M 8 | 10 9 | 10 | 11 | 12 | 8123 13 | 14 | 17 | 18 | 19 | 20 | 21 | /etc/clickhouse-server/server.crt 22 | /etc/clickhouse-server/server.key 23 | 24 | /etc/clickhouse-server/dhparam.pem 25 | none 26 | true 27 | true 28 | sslv2,sslv3 29 | true 30 | 31 | 32 | true 33 | true 34 | sslv2,sslv3 35 | true 36 | 37 | 38 | 39 | RejectCertificateHandler 40 | 41 | 42 | 43 | 44 | 45 | 48 | 49 | 9000 50 | 51 | 52 | 9009 53 | 54 | 58 | 61 | 62 | 63 | 64 | ::1 65 | 127.0.0.1 66 | 67 | 4096 68 | 3 69 | 70 | 71 | 100 72 | 73 | 75 | 76 | 77 | 82 | 8589934592 83 | 84 | 88 | 5368709120 89 | 90 | 91 | 92 | /var/lib/clickhouse/ 93 | 94 | 95 | /var/lib/clickhouse/tmp/ 96 | 97 | 98 | users.xml 99 | 100 | 101 | default 102 | 103 | 104 | default 105 | 106 | 115 | 116 | 117 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 1 129 | 130 | false 131 | 132 | example01-01-1 133 | 9000 134 | 135 | 136 | example01-01-2 137 | 9000 138 | 139 | 140 | 141 | 2 142 | false 143 | 144 | example01-02-1 145 | 9000 146 | 147 | 148 | example01-02-2 149 | 9000 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 161 | 162 | 167 | 168 | 169 | 174 | 175 | 176 | 177 | 178 | 3600 179 | 180 | 181 | 182 | 3600 183 | 184 | 185 | 60 186 | 187 | 188 | 195 | 219 | 220 | 221 | 222 | 223 | 227 | system 228 | query_log
229 | 230 | 231 | 7500 232 | 233 | 234 | 235 | 243 | 244 | 245 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 259 | *_dictionary.xml 260 | 261 | 264 | 265 | 277 | 278 | 279 | 280 | /clickhouse/task_queue 281 | 282 | 283 | 285 | 286 | 287 | /clickhouse/task_queue/ddl 288 | 289 | 290 | 291 | 296 | 297 | 303 | 304 | 305 | 306 | 307 | 308 | click_cost 309 | any 310 | 311 | 0 312 | 3600 313 | 314 | 315 | 86400 316 | 60 317 | 318 | 319 | 320 | max 321 | 322 | 0 323 | 60 324 | 325 | 326 | 3600 327 | 300 328 | 329 | 330 | 86400 331 | 3600 332 | 333 | 334 | 335 | 336 | -------------------------------------------------------------------------------- /config/config.xml.old: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | trace 5 | /var/log/clickhouse-server/clickhouse-server.log 6 | /var/log/clickhouse-server/clickhouse-server.err.log 7 | 1000M 8 | 10 9 | 10 | 11 | 12 | 8123 13 | 14 | 17 | 18 | 19 | 20 | 21 | /etc/clickhouse-server/server.crt 22 | /etc/clickhouse-server/server.key 23 | 24 | /etc/clickhouse-server/dhparam.pem 25 | none 26 | true 27 | true 28 | sslv2,sslv3 29 | true 30 | 31 | 32 | true 33 | true 34 | sslv2,sslv3 35 | true 36 | 37 | 38 | 39 | RejectCertificateHandler 40 | 41 | 42 | 43 | 44 | 45 | 48 | 49 | 9000 50 | 51 | 52 | 9009 53 | 54 | 58 | 61 | 62 | 63 | 64 | ::1 65 | 127.0.0.1 66 | 67 | 4096 68 | 3 69 | 70 | 71 | 100 72 | 73 | 75 | 76 | 77 | 82 | 8589934592 83 | 84 | 88 | 5368709120 89 | 90 | 91 | 92 | /var/lib/clickhouse/ 93 | 94 | 95 | /var/lib/clickhouse/tmp/ 96 | 97 | 98 | users.xml 99 | 100 | 101 | default 102 | 103 | 104 | default 105 | 106 | 115 | 116 | 117 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 1 129 | 130 | false 131 | 132 | example01-01-1 133 | 9000 134 | 135 | 136 | example01-01-2 137 | 9000 138 | 139 | 140 | 141 | 2 142 | false 143 | 144 | example01-02-1 145 | 9000 146 | 147 | 148 | example01-02-2 149 | 9000 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 161 | 162 | 167 | 168 | 169 | 174 | 175 | 176 | 177 | 178 | 3600 179 | 180 | 181 | 182 | 3600 183 | 184 | 185 | 60 186 | 187 | 188 | 195 | 219 | 220 | 221 | 222 | 223 | 227 | system 228 | query_log
229 | 230 | 231 | 7500 232 | 233 | 234 | 235 | 243 | 244 | 245 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 259 | *_dictionary.xml 260 | 261 | 264 | 265 | 277 | 278 | 279 | 280 | /clickhouse/task_queue 281 | 282 | 283 | 285 | 286 | 287 | /clickhouse/task_queue/ddl 288 | 289 | 290 | 291 | 296 | 297 | 303 | 304 | 305 | 306 | 307 | 308 | click_cost 309 | any 310 | 311 | 0 312 | 3600 313 | 314 | 315 | 86400 316 | 60 317 | 318 | 319 | 320 | max 321 | 322 | 0 323 | 60 324 | 325 | 326 | 3600 327 | 300 328 | 329 | 330 | 86400 331 | 3600 332 | 333 | 334 | 335 | 336 | -------------------------------------------------------------------------------- /config/config.xml.test: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /config/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 10000000000 9 | 10 | 11 | 0 12 | 13 | 20 | random 21 | 22 | 23 | 24 | 25 | 1 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 46 | 47 | 48 | 67 | 68 | ::/0 69 | 70 | 71 | 72 | default 73 | 74 | 75 | default 76 | 77 | 78 | 79 | 80 | 81 | 82 | ::1 83 | 127.0.0.1 84 | 85 | readonly 86 | default 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 3600 98 | 99 | 100 | 0 101 | 0 102 | 0 103 | 0 104 | 0 105 | 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /package_clear_old.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TO_DEL="build dist clickhouse_mysql.egg-info" 4 | 5 | echo "########################################" 6 | echo "### Clear all build and release data ###" 7 | echo "########################################" 8 | 9 | echo "Deleting:" 10 | for DEL in $TO_DEL; do 11 | echo " $DEL" 12 | done 13 | 14 | echo "rm -rf $TO_DEL" 15 | rm -rf $TO_DEL 16 | -------------------------------------------------------------------------------- /package_publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "###########################" 4 | echo "### Publish from dist/* ###" 5 | echo "###########################" 6 | 7 | echo "Going to publish:" 8 | for FILE in $(ls dist/*); do 9 | echo " $FILE" 10 | done 11 | 12 | twine upload dist/* 13 | -------------------------------------------------------------------------------- /package_source_distr.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ./package_clear_old.sh 4 | 5 | python3 setup.py sdist 6 | -------------------------------------------------------------------------------- /package_wheels_distr.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ./package_clear_old.sh 4 | 5 | python3 setup.py bdist_wheel 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | paramiko 2 | lxml 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from setuptools import setup, find_packages 5 | 6 | setup( 7 | name="clickhouse-manager", 8 | 9 | # version should comply with PEP440 10 | version='0.0.201801091', 11 | 12 | description='ClickHouse Manager', 13 | long_description='ClickHouse Manager', 14 | 15 | # homepage 16 | url="https://github.com/altinity/clickhouse-cluster-manager", 17 | 18 | author="Vladislav Klimenko", 19 | author_email="sunsingerus@gmail.com", 20 | 21 | license="MIT", 22 | 23 | # see https://pypi.python.org/pypi?%3Aaction=list_classifiers 24 | classifiers=[ 25 | # How mature is this project? Common values are 26 | # 3 - Alpha 27 | # 4 - Beta 28 | # 5 - Production/Stable 29 | 'Development Status :: 3 - Alpha', 30 | 31 | 'Intended Audience :: Developers', 32 | 'Topic :: Database', 33 | 34 | # should match license above 35 | 'License :: OSI Approved :: MIT License', 36 | 37 | # supported Python versions 38 | 'Programming Language :: Python :: 3', 39 | 'Programming Language :: Python :: 3.5', 40 | 'Programming Language :: Python :: 3.6', 41 | ], 42 | 43 | # what does the project relate to? 44 | keywords='clickhouse manager', 45 | 46 | # list of packages to be included into project 47 | packages=find_packages(exclude=[ 48 | 'contrib', 49 | 'docs', 50 | 'tests', 51 | ]), 52 | 53 | # list of additional package data to be attached to packages 54 | package_data={ 55 | }, 56 | 57 | # run-time dependencies 58 | # these will be installed by pip 59 | # https://packaging.python.org/en/latest/requirements.html 60 | install_requires=[ 61 | 'paramiko', 62 | 'lxml', 63 | ], 64 | 65 | # cross-platform support for pip to create the appropriate form of executable 66 | entry_points={ 67 | 'console_scripts': [ 68 | # executable name=what to call 69 | 'clickhouse-manager=clickhouse_manager:main', 70 | ], 71 | }, 72 | 73 | # python_requires='>=3.3', 74 | ) 75 | --------------------------------------------------------------------------------