├── MANIFEST.in ├── ssl ├── keystore.jks ├── cassandra.crt ├── truststore.jks ├── cassandra.pem ├── client_cert.pem └── client_key.pem ├── .gitignore ├── ccmlib ├── cmds │ ├── common.py │ ├── __init__.py │ └── command.py ├── __init__.py ├── hcd │ ├── __init__.py │ ├── hcd_cluster.py │ └── hcd_node.py ├── dse │ ├── __init__.py │ └── dse_cluster.py ├── extension.py ├── cluster_factory.py ├── remote.py └── repository.py ├── setup.cfg ├── tests ├── requirements.txt ├── __init__.py ├── conftest.py ├── ccmtest.py ├── test_common.py └── test_lib.py ├── requirements.txt ├── setup.py ├── .asf.yaml ├── README-CASSANDRA-17379.md ├── misc ├── ccm-macos.bash ├── ccm-completion.bash └── Vagrantfile ├── NETWORK_ALIASES.md ├── NOTICE ├── .github └── workflows │ ├── main-python-2-7.yml │ └── main.yml ├── ccm ├── INSTALL.md ├── README.md └── license.txt /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | -------------------------------------------------------------------------------- /ssl/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/cassandra-ccm/HEAD/ssl/keystore.jks -------------------------------------------------------------------------------- /ssl/cassandra.crt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/cassandra-ccm/HEAD/ssl/cassandra.crt -------------------------------------------------------------------------------- /ssl/truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/cassandra-ccm/HEAD/ssl/truststore.jks -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | build/ 4 | dist/ 5 | venv/ 6 | ccm.iml 7 | ccm.py 8 | .idea/ 9 | ccm.egg-info/ 10 | .project 11 | .tox/ 12 | .coverage 13 | dse_creds.txt 14 | .eggs/ 15 | junit.xml 16 | /nbproject/ 17 | -------------------------------------------------------------------------------- /ccmlib/cmds/common.py: -------------------------------------------------------------------------------- 1 | from ccmlib.cmds import cluster_cmds, command, node_cmds 2 | 3 | 4 | def get_command(kind, cmd): 5 | cmd_name = kind.lower().capitalize() + cmd.lower().capitalize() + "Cmd" 6 | try: 7 | klass = (cluster_cmds if kind.lower() == 'cluster' else node_cmds).__dict__[cmd_name] 8 | except KeyError: 9 | return None 10 | if not issubclass(klass, command.Cmd): 11 | return None 12 | return klass() 13 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name=ccm 3 | version=3.1.6 4 | description=Cassandra Cluster Manager 5 | url='https://github.com/apache/cassandra-ccm' 6 | description-file = README.md 7 | description-content-type = text/markdown; charset=UTF-8 8 | classifier= 9 | "License :: OSI Approved :: Apache Software License", 10 | "Programming Language :: Python", 11 | 'Programming Language :: Python :: 2', 12 | 'Programming Language :: Python :: 2.7', 13 | 'Programming Language :: Python :: 3', 14 | 'Programming Language :: Python :: 3.8', 15 | 'Programming Language :: Python :: 3.11' 16 | 17 | [pbr] 18 | skip_authors=1 19 | skip_changelog=1 20 | 21 | [files] 22 | packages = 23 | ccmlib 24 | ccmlib.cmds 25 | 26 | -------------------------------------------------------------------------------- /ccmlib/cmds/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | mock==5.1.0 18 | pytest==8.2.1 19 | requests==2.32.2 20 | pylint==3.2.2 -------------------------------------------------------------------------------- /ccmlib/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # load __init__.py from cluster type plugin implementation packages 18 | from ccmlib import dse, hcd 19 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | pyYaml<5.4; python_version < '3' 18 | pyYaml; python_version >= '3' 19 | six >=1.4.1 20 | psutil 21 | 22 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import sys 19 | 20 | 21 | TEST_DIR = "test-dir" 22 | 23 | 24 | if sys.version_info < (3, 0): 25 | FileNotFoundError = OSError 26 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import os 19 | import pytest 20 | import shutil 21 | 22 | from tests import TEST_DIR 23 | 24 | @pytest.fixture(scope="package", autouse=True) 25 | def setup_and_package(): 26 | try: 27 | shutil.rmtree(TEST_DIR) 28 | except FileNotFoundError: 29 | pass 30 | 31 | os.makedirs(TEST_DIR) 32 | yield 33 | shutil.rmtree(TEST_DIR) 34 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | 20 | from platform import system 21 | from shutil import copyfile 22 | 23 | try: 24 | from setuptools import setup 25 | except ImportError: 26 | from distutils.core import setup 27 | 28 | ccmscript = 'ccm' 29 | 30 | if system() == "Windows": 31 | copyfile('ccm', 'ccm.py') 32 | ccmscript = 'ccm.py' 33 | 34 | setup( 35 | setup_requires=['pbr>=5.8.1'], 36 | scripts=[ccmscript], 37 | pbr=True, 38 | ) 39 | -------------------------------------------------------------------------------- /ssl/cassandra.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDczCCAlugAwIBAgIEfxTaWTANBgkqhkiG9w0BAQsFADBqMQswCQYDVQQGEwJU 3 | RTELMAkGA1UECBMCQ0ExFDASBgNVBAcTC1NhbnRhIENsYXJhMREwDwYDVQQKEwhE 4 | YXRhU3RheDELMAkGA1UECxMCVEUxGDAWBgNVBAMTD1BoaWxpcCBUaG9tcHNvbjAe 5 | Fw0xNjAyMDgwMTMyMThaFw0xODAyMDcwMTMyMThaMGoxCzAJBgNVBAYTAlRFMQsw 6 | CQYDVQQIEwJDQTEUMBIGA1UEBxMLU2FudGEgQ2xhcmExETAPBgNVBAoTCERhdGFT 7 | dGF4MQswCQYDVQQLEwJURTEYMBYGA1UEAxMPUGhpbGlwIFRob21wc29uMIIBIjAN 8 | BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAgh3t98zq+vlHPoxD+opzknJ8+32B 9 | kPYH5OsgJYPgG5DHsJifZ9wg4z1eMdhNHMXGhpjDC/2a9RP+zgLIrz2qiF80k3U7 10 | 4YcthzFOjjQe6otHefZRrm/wWN9roEzuX+jDTHshywrWqk6n8AQ1P2XAyQ+JXqE/ 11 | 8g3kp13ISw+iyn06OGCzGpS8sXMyjM8dqyEXMYXRgVcsGyhnOry0AUub0SRYIGIX 12 | n8cO9THSMAfyazjP17JopYjBnbGx4q3+G7AIQ7LV/Po2k2/xuGPErJq5EY+8kovL 13 | 5XbOgSOJwWe6wN9jpdXZCI8Uu1nvpDnPSrQ8ULZYNvxIsHyfawlV1q16DwIDAQAB 14 | oyEwHzAdBgNVHQ4EFgQUL/pt5vT63nQ4isRbjmFQmjE+gXMwDQYJKoZIhvcNAQEL 15 | BQADggEBAD47rYXdiTYbUECvD5x76IpEmxyuRRjEDtaoiP+i5pd7SLGGwTaWy7N2 16 | h3vjawbW+NNHdSQPReoZ6aihyS0hKo3kInhClScgMsdAWgjp+TUTRvJ+w8K2QfL6 17 | Kv5NJM462H0nGC0Hd7jk7B++GyhjrPGluUnV+FiyjiEHOKNpTxAmUjUUgTSJXCTL 18 | 71BSrwOL3TYk5QbVQMsVAfLSoQjYkbb0SRSWOBnrXglq4MWP4fiJ180pae/pK73O 19 | FQqUL1jANHw4lNQfKxLKpoTNzcO+a9pbOF3CXuVueETvmvj9KcU1ksWEgMY4i8XP 20 | ZmcxuiAEQ+kL+EXyeHFV8vU3LfI0Vwo= 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /ccmlib/hcd/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | from ccmlib import extension 19 | from ccmlib.cmds.cluster_cmds import ClusterCreateCmd 20 | from ccmlib.hcd.hcd_cluster import isHcdClusterType 21 | 22 | 23 | # static initialisation: register the extension cluster type, add hcd specific option to ClusterCreateCmd 24 | 25 | extension.CLUSTER_TYPES.append(isHcdClusterType) 26 | 27 | ClusterCreateCmd.options_list.extend([ 28 | (["--hcd"], {'action': "store_true", 'dest': "hcd", 'help': "Use with -v or --install-dir to indicate that the version being loaded is HCD"})]) 29 | -------------------------------------------------------------------------------- /.asf.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | notifications: 19 | commits: commits@cassandra.apache.org 20 | issues: commits@cassandra.apache.org 21 | pullrequests: pr@cassandra.apache.org 22 | jira_options: link worklog 23 | 24 | github: 25 | description: "Apache Cassandra® Cluster Manager (CCM) – easily create and destroy clusters on localhost" 26 | homepage: https://cassandra.apache.org/ 27 | enabled_merge_buttons: 28 | squash: false 29 | merge: false 30 | rebase: true 31 | features: 32 | wiki: false 33 | issues: true 34 | projects: false 35 | discussions: false 36 | autolink_jira: 37 | - CASSANDRA 38 | protected_branches: 39 | trunk: 40 | required_linear_history: true 41 | -------------------------------------------------------------------------------- /ssl/client_cert.pem: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: client 3 | localKeyID: 54 69 6D 65 20 31 34 35 34 38 39 35 31 34 30 35 36 32 4 | subject=/C=TE/ST=CA/L=Santa Clara/O=DataStax/OU=TE/CN=Philip Thompson 5 | issuer=/C=TE/ST=CA/L=Santa Clara/O=DataStax/OU=TE/CN=Philip Thompson 6 | -----BEGIN CERTIFICATE----- 7 | MIIDczCCAlugAwIBAgIEcXcM5DANBgkqhkiG9w0BAQsFADBqMQswCQYDVQQGEwJU 8 | RTELMAkGA1UECBMCQ0ExFDASBgNVBAcTC1NhbnRhIENsYXJhMREwDwYDVQQKEwhE 9 | YXRhU3RheDELMAkGA1UECxMCVEUxGDAWBgNVBAMTD1BoaWxpcCBUaG9tcHNvbjAe 10 | Fw0xNjAyMDgwMTMyMjBaFw0xODAyMDcwMTMyMjBaMGoxCzAJBgNVBAYTAlRFMQsw 11 | CQYDVQQIEwJDQTEUMBIGA1UEBxMLU2FudGEgQ2xhcmExETAPBgNVBAoTCERhdGFT 12 | dGF4MQswCQYDVQQLEwJURTEYMBYGA1UEAxMPUGhpbGlwIFRob21wc29uMIIBIjAN 13 | BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhbUk95AEcexCz2xwn22cDR1SBepp 14 | WUEG8H49C0S9QG8o4DX6vPJaKDCESLNWQxuOd30KGlY19nPwPuhikAdnCohzxj+l 15 | JrEAMSmCugi6g95xne+tHx+zIIR1pvJ+YtAw11OulZNcZArt1FWniPjKz4OTFnEl 16 | otrj6W4cLz9TMiFrvvDCqwnJv9Y8Wi5jgW7zoLhGWX/TYZUimSFRAKxo7mwiAMUb 17 | u9WPTt+zmP3n5/oHhdnnnjP84Wrdq0aaZ6K6KwsuGdnuXub22XRD82FT2EWrYLoM 18 | GjwzwOQ7ldtCYy6AjKpQIQxiI2U/X1fvW50XKocNmJzbBIt46QmCZktaJwIDAQAB 19 | oyEwHzAdBgNVHQ4EFgQURNY1taeNOSYhMNYjkv/n0Thmvc4wDQYJKoZIhvcNAQEL 20 | BQADggEBAFfPD3i0Csu99AqCeHWZGII93Px9uuGryYkwTBh2n82Vcwn12LjvhrdJ 21 | CR/UN2eSnfZdg3kK8791TLm5pBqVXbsR+Melmj1ocjNJdIqqrD6UuCoJMMPi21k2 22 | IT6vnsiMsB+yHP88N/klf5Uc8G6Mz/949GccY00dd756Hmp1rnkjoa0bo2pqOFQh 23 | xQSEVF0fyvbsXoJeS7AZoIlAAHciUOd/Sd9V6tZ43BC97Vwn+UR2+sH6v8Vxd13L 24 | q1/+JDXi9PwDtw1kECjpH4HPiY/ei9r1fuy+Lp8OC34dPWZtJ31xsh5jWrthhKaz 25 | AqOccZ9cDUNECWTMXHCfcB+w0cU9ONM= 26 | -----END CERTIFICATE----- 27 | -------------------------------------------------------------------------------- /tests/ccmtest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import os 19 | from unittest import TestCase 20 | 21 | try: 22 | FileNotFoundError 23 | except NameError: 24 | FileNotFoundError = IOError # Python 2.7 compatibility 25 | 26 | class Tester(TestCase): 27 | 28 | check_log_errors = True 29 | 30 | def __init__(self, *argv, **kwargs): 31 | super(Tester, self).__init__(*argv, **kwargs) 32 | 33 | def setUp(self): 34 | self.check_log_errors = True 35 | 36 | def tearDown(self): 37 | if hasattr(self, 'cluster'): 38 | try: 39 | if self.check_log_errors: 40 | for node in self.cluster.nodelist(): 41 | try: 42 | self.assertListEqual(node.grep_log_for_errors(), []) 43 | except FileNotFoundError: 44 | continue 45 | finally: 46 | test_path = self.cluster.get_path() 47 | self.cluster.remove() 48 | if os.path.exists(test_path): 49 | os.remove(test_path) 50 | -------------------------------------------------------------------------------- /ssl/client_key.pem: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: client 3 | localKeyID: 54 69 6D 65 20 31 34 35 34 38 39 35 31 34 30 35 36 32 4 | Key Attributes: 5 | -----BEGIN PRIVATE KEY----- 6 | MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCFtST3kARx7ELP 7 | bHCfbZwNHVIF6mlZQQbwfj0LRL1AbyjgNfq88looMIRIs1ZDG453fQoaVjX2c/A+ 8 | 6GKQB2cKiHPGP6UmsQAxKYK6CLqD3nGd760fH7MghHWm8n5i0DDXU66Vk1xkCu3U 9 | VaeI+MrPg5MWcSWi2uPpbhwvP1MyIWu+8MKrCcm/1jxaLmOBbvOguEZZf9NhlSKZ 10 | IVEArGjubCIAxRu71Y9O37OY/efn+geF2eeeM/zhat2rRppnororCy4Z2e5e5vbZ 11 | dEPzYVPYRatgugwaPDPA5DuV20JjLoCMqlAhDGIjZT9fV+9bnRcqhw2YnNsEi3jp 12 | CYJmS1onAgMBAAECggEABqezhVb3wavez3A4Utcj00tIT98RC04/SC0gYLU1LkXa 13 | JP7K0ijF8AYqL2wtuP1gI/ZnUFRGL1Qp+xeaAE0+Bbow+Qcl8z0QI2JLjXLtxa6G 14 | vTO2zDvJsK6nJH4haE2wgKc7o1pIWPpqSA1TX2/yuE12PsG2+9olSfMfGALw4yfQ 15 | RyC4HFCJdFZ3sPvwtbBxX60ZsZAanRMy22+iRJQ9qGLdf3rndEjkR70Ls/wOLcox 16 | TJM6/98rvIxnMuDLhNYnQdjqLa5gICPtxzVdELzoKqFPDI1zKNZAX9fOINRwMKqr 17 | Wt0fUyv2crqrJ8nQlRdsRGYxGJT5NRlbrzJz0XLUMQKBgQC9BFwJjLJBZZP87WVb 18 | /kWleWeDPvuraTJkxnXcLLnHxb/CEK23wq93katCMv5TSIiUlaUoHzK46yBJOPgB 19 | 7DvpZ+F8ub0soYOeSBtMtYgsdYI5OQOy+qxPgAy6V8Xq8qvPeGZa1t4bLQ3m+o8a 20 | wbA8vC1sxFJEVWfpo1dhA4whOwKBgQC1FxvEftIiu0P41iI0vt9LUag82gWiJhNT 21 | Wt8ee9qYokaEe3B4SNtWAJfXf3HVdYGHwOt7Shiu34V1lP7UT/MhIBGtFz1rXzVd 22 | B8MMbE9uDbufhvJ8aqJ5g4kHFCGBGpYnJh8O2dql3j0kZ4ia7Rzu2YRcVZ68MQ53 23 | NsYkOADcBQKBgA3CmnyrfHKcVXitQ5q92Q4h13JLjIC/CMcjV4Mu0luDvuD+29ar 24 | 2qBGv4PzOGaRujeu6TYRh1zE6TXLauqg6v+j61tsHiR4oZ9NOoeME5zA9Tj7OJS4 25 | AQFMniCWsTbYcb+J0VG9oK+zyPZOuUpGXXEedeQcKq0E1qrAlGTgoDvvAoGABobQ 26 | r2JKvIm3R26gSPpgHdzRjW/mKBPrOmPaCsU3+axPklLImO03SoA2+MNVHPZhNr1T 27 | P1xKS3Mu4i/+hzRidN0tBeoCgq4pxDKVaws0SakhC/zXHHjTZkHBXInzMy38H2kW 28 | UXi0kqnR1lAM8lh9ZHZeeN11HR8/gDhvJ7sE/OkCgYAYS0QLA00gDNM0J5+8+4KH 29 | ET2SUlWKBv+vWAj9mXVLdg+7KS4HIHR715vAyIV7HNVXohj9LbWYsMG2RGkbCqYO 30 | Hrrqnulet09T9Z+HD008s9xGnGFU9necujIQV0uHz1j4lCnOgVoN2NiwtU8H0dI4 31 | SumcCsPe4DQ6Y5TTU1WIlg== 32 | -----END PRIVATE KEY----- 33 | -------------------------------------------------------------------------------- /README-CASSANDRA-17379.md: -------------------------------------------------------------------------------- 1 | CCM (Cassandra Cluster Manager) 2 | CASSANDRA-17379 README 3 | ==================================================== 4 | 5 | 6 | ----  7 |   8 | 9 | 10 | WARNING - CCM configuration changes using updateconf does not happen according to CASSANDRA-17379 11 | ------------------------------------------------------------------------------------------------- 12 | 13 | After CASSANDRA-15234, to support the Python upgrade tests CCM updateconf is replacing 14 | new key name and value in case the old key name and value is provided. 15 | For example, if you add to config `permissions_validity_in_ms`, it will replace 16 | `permissions_validity` in default cassandra.yaml 17 | This was needed to ensure correct overloading as CCM cassandra.yaml has keys 18 | sorted lexicographically. CASSANDRA-17379 was opened to improve the user experience 19 | and deprecate the overloading of parameters in cassandra.yaml. In CASSANDRA 4.1+, by default, 20 | we refuse starting Cassandra with a config containing both old and new config keys for the 21 | same parameter. Start Cassandra with `-Dcassandra.allow_new_old_config_keys=true` to override. 22 | For historical reasons duplicate config keys in cassandra.yaml are allowed by default, start 23 | Cassandra with `-Dcassandra.allow_duplicate_config_keys=false` to disallow this. Please note 24 | that key_cache_save_period, row_cache_save_period, counter_cache_save_period will be affected 25 | only by `-Dcassandra.allow_duplicate_config_keys`. Ticket CASSANDRA-17949 was opened to decide 26 | the future of CCM updateconf post CASSANDRA-17379, until then - bear in mind that old replace 27 | new parameters' in cassandra.yaml when using updateconf even if 28 | `-Dcassandra.allow_new_old_config_keys=false` is set by default. 29 | 30 | TLDR Do not exercise overloading of parameters in CCM if possible. Also, the mentioned changes 31 | are done only in master branch. Probably the best way to handle cassandra 4.1 in CCM at this 32 | point is to set `-Dcassandra.allow_new_old_config_keys=false` and 33 | `-Dcassandra.allow_duplicate_config_keys=false` 34 | to prohibit any kind of overloading when using CCM master and CCM released versions 35 | -------------------------------------------------------------------------------- /misc/ccm-macos.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | CLUSTER_NAME=${CLUSTER_NAME:-test} 19 | NUM_NODES=${NUM_NODES:-3} 20 | CASSANDRA_DIR=${CASSANDRA_DIR:-$HOME/cassandra} 21 | PYTHON_VERSION=${PYTHON_VERSION:-python3} 22 | 23 | function fail() 24 | { 25 | echo "ERROR: $1" 26 | exit 1 27 | } 28 | 29 | 30 | if [ ! -z $VIRTUAL_ENV ]; then 31 | if [ ! -d ${CASSANDRA_DIR}/pylib/venv ]; then 32 | 33 | if [ "${PYTHON_VERSION}" != "python3" -a "${PYTHON_VERSION}" != "python2" ]; then 34 | fail "Specify Python version python3 or python2" 35 | fi 36 | 37 | # Initialize the virtualenv used for Cassandra's pylib 38 | virtualenv --python=$PYTHON_VERSION ${CASSANDRA_DIR}/pylib/venv 39 | source ${CASSANDRA_DIR}/pylib/venv/bin/activate 40 | pip install -r ${CASSANDRA_DIR}/pylib/requirements.txt 41 | pip freeze 42 | else 43 | # use Cassandra's pylib virtual environment 44 | source ${CASSANDRA_DIR}/pylib/venv/bin/activate 45 | fi 46 | fi 47 | 48 | ccm remove $CLUSTER_NAME 49 | 50 | for i in $(seq 1 $(($NUM_NODES - 1))); 51 | do 52 | if=127.0.0.$((i + 1)) 53 | echo $if 54 | if ! ifconfig lo0 | grep "$if" > /dev/null; then 55 | echo "Configuring interface $if" 56 | sudo ifconfig lo0 alias $if || fail "Unable to configure interface $if" 57 | fi 58 | done 59 | 60 | ccm create $CLUSTER_NAME -n $NUM_NODES --install-dir=${CASSANDRA_DIR} 61 | ccm populate -d -n $NUM_NODES 62 | ccm start 63 | -------------------------------------------------------------------------------- /ccmlib/dse/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | from ccmlib import extension 19 | from ccmlib import common 20 | from ccmlib.cmds.cluster_cmds import ClusterAddCmd, ClusterCreateCmd 21 | from ccmlib.dse.dse_cluster import isDseClusterType 22 | 23 | 24 | # static initialisation: register the extension cluster type, add dse specific options to ClusterCreateCmd and ClusterAddCmd 25 | extension.CLUSTER_TYPES.append(isDseClusterType) 26 | 27 | ClusterCreateCmd.options_list.extend([ 28 | (['-o', "--opsc"], {'type': "string", 'dest': "opscenter", 'help': "Download and use provided OpsCenter version to install with DSE. Will have no effect on cassandra installs)", 'default': None}), 29 | (["--dse"], {'action': "store_true", 'dest': "dse", 'help': "Use with -v or --install-dir to indicate that the version being loaded is DSE"}), 30 | (["--dse-username"], {'type': "string", 'dest': "dse_username", 'help': "The username to use to download DSE with", 'default': None}), 31 | (["--dse-password"], {'type': "string", 'dest': "dse_password", 'help': "The password to use to download DSE with", 'default': None}), 32 | (["--dse-credentials"], {'type': "string", 'dest': "dse_credentials_file", 'help': "An ini-style config file containing the dse_username and dse_password under a dse_credentials section. [default to {}/.dse.ini if it exists]".format(common.get_default_path_display_name()), 'default': None})]) 33 | 34 | ClusterAddCmd.options_list.append( (['--dse'], {'action': "store_true", 'dest': "dse_node", 'help': "Add node to DSE Cluster", 'default': False})) 35 | -------------------------------------------------------------------------------- /ccmlib/extension.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | CLUSTER_TYPES = [] 19 | PRE_CLUSTER_START_HOOKS = [] 20 | POST_CLUSTER_START_HOOKS = [] 21 | PRE_CLUSTER_STOP_HOOKS = [] 22 | POST_CLUSTER_STOP_HOOKS = [] 23 | APPEND_TO_CLUSTER_CONFIG_HOOKS = [] 24 | LOAD_FROM_CLUSTER_CONFIG_HOOKS = [] 25 | APPEND_TO_SERVER_ENV_HOOKS = [] 26 | APPEND_TO_CLIENT_ENV_HOOKS = [] 27 | APPEND_TO_CQLSH_ARGS_HOOKS = [] 28 | 29 | def get_cluster_class(install_dir, options=None): 30 | for is_cluster_type in CLUSTER_TYPES: 31 | cluster_class = is_cluster_type(install_dir, options) 32 | if cluster_class: 33 | return cluster_class 34 | from ccmlib import cluster 35 | return cluster.Cluster 36 | 37 | def pre_cluster_start(cluster): 38 | for hook in PRE_CLUSTER_START_HOOKS: 39 | hook(cluster) 40 | 41 | 42 | def post_cluster_start(cluster): 43 | for hook in POST_CLUSTER_START_HOOKS: 44 | hook(cluster) 45 | 46 | 47 | def pre_cluster_stop(cluster): 48 | for hook in PRE_CLUSTER_STOP_HOOKS: 49 | hook(cluster) 50 | 51 | 52 | def post_cluster_stop(cluster): 53 | for hook in POST_CLUSTER_STOP_HOOKS: 54 | hook(cluster) 55 | 56 | 57 | def append_to_cluster_config(cluster, config_map): 58 | for hook in APPEND_TO_CLUSTER_CONFIG_HOOKS: 59 | hook(cluster, config_map) 60 | 61 | 62 | def load_from_cluster_config(cluster, config_map): 63 | for hook in LOAD_FROM_CLUSTER_CONFIG_HOOKS: 64 | hook(cluster, config_map) 65 | 66 | 67 | def append_to_server_env(node, env): 68 | for hook in APPEND_TO_SERVER_ENV_HOOKS: 69 | hook(node, env) 70 | 71 | 72 | def append_to_client_env(node, env): 73 | for hook in APPEND_TO_CLIENT_ENV_HOOKS: 74 | hook(node, env) 75 | 76 | 77 | def append_to_cqlsh_args(node, env, args): 78 | for hook in APPEND_TO_CQLSH_ARGS_HOOKS: 79 | hook(node, env, args) 80 | -------------------------------------------------------------------------------- /NETWORK_ALIASES.md: -------------------------------------------------------------------------------- 1 | Network aliases required for CCM 2 | -------------------------------- 3 | CCM needs a local ip address for each Cassandra node it creates. By default CCM assumes each node's ip address is 127.0.0.x, 4 | where x is the node id. 5 | 6 | For example if you populated your cluster with 3 nodes, create interfaces for 127.0.0.2 and 127.0.0.3 7 | (the first node of course uses 127.0.0.1). 8 | 9 | Relevant CCM logs: 10 | 11 | * Missing an alias: 12 | 13 | ``` 14 | (...) Inet address 127.0.0.2:9042 is not available; a cluster may already be running or you may need to add the loopback alias 15 | ``` 16 | 17 | * Another node is already using a specific alias: 18 | 19 | ``` 20 | (...) Inet address 127.0.0.2:9042 is not available: [Errno 48] Address already in use 21 | ``` 22 | 23 | Mac OSX temporary network aliases 24 | --------------------------------- 25 | To get up and running right now, create a temporary alias for every node except the first: 26 | 27 | ``` 28 | sudo ifconfig lo0 alias 127.0.0.2 29 | sudo ifconfig lo0 alias 127.0.0.3 30 | ``` 31 | Note that these aliases are only temporary and will disappear on reboot. 32 | 33 | Mac OSX persistent network aliases 34 | ---------------------------------- 35 | Persist network aliases if you use CCM often or so infrequently you forget this step. 36 | 37 | To persist a network alias on Mac OSX you need to create a RunAtLoad launch daemon which OSX automatically loads on startup. For example: 38 | 39 | Create a shell script: 40 | ``` 41 | sudo vim /Library/LaunchDaemons/com.ccm.lo0.alias.sh 42 | ``` 43 | 44 | Contents of the script: 45 | ``` 46 | #!/bin/sh 47 | sudo /sbin/ifconfig lo0 alias 127.0.0.2; 48 | sudo /sbin/ifconfig lo0 alias 127.0.0.3; 49 | ``` 50 | 51 | Set access of the script: 52 | ``` 53 | sudo chmod 755 /Library/LaunchDaemons/com.ccm.lo0.alias.sh 54 | ``` 55 | 56 | Create a plist to launch the script: 57 | ``` 58 | sudo vim /Library/LaunchDaemons/com.ccm.lo0.alias.plist 59 | ``` 60 | 61 | Contents of the plist: 62 | ``` 63 | 64 | 65 | 66 | 67 | Label 68 | com.ccm.lo0.alias 69 | RunAtLoad 70 | 71 | ProgramArguments 72 | 73 | /Library/LaunchDaemons/com.ccm.lo0.alias.sh 74 | 75 | StandardErrorPath 76 | /var/log/loopback-alias.log 77 | StandardOutPath 78 | /var/log/loopback-alias.log 79 | 80 | 81 | ``` 82 | 83 | Set access of the plist: 84 | ``` 85 | sudo chmod 0644 /Library/LaunchDaemons/com.ccm.lo0.alias.plist 86 | sudo chown root:staff /Library/LaunchDaemons/com.ccm.lo0.alias.plist 87 | ``` 88 | 89 | Launch the daemon now. OSX will automatically reload it on startup. 90 | ``` 91 | sudo launchctl load /Library/LaunchDaemons/com.ccm.lo0.alias.plist 92 | ``` 93 | 94 | Verify you can ping 127.0.0.2 and 127.0.0.3. 95 | 96 | If you ever want to permanently kill the daemon, simply delete its plist from /Library/LaunchDaemons/. 97 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache Cassandra Cluster Manager 2 | Copyright 2014 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | 8 | This product originates, before git sha 9 | cc8cb98b29f68d12c8b5fa093a33dc3b346873b3, from software from DataStax 10 | and other individual contributors. 11 | 12 | Non-DataStax contributors are listed below. Those marked with 13 | asterisk have agreed to donate (copyright assign) their contributions to the 14 | Apache Software Foundation, signing CLAs when appropriate. 15 | 16 | Lucas Meneghel @lmr * 17 | Andrew Hust @nutbunnies 18 | Jaume Marhuenda @beltran 19 | Daniel Compton @danielcompton * 20 | ??? @steveandwang 21 | Byron Clark @byronclark * 22 | Thibault Charbonnier @thibaultcha * 23 | ??? @c-kodman 24 | Sandeep Tamhankar @stamhankar999 25 | Tomek Bujok @tombujok 26 | Pekka Enberg @penberg * 27 | Brian Weck @bweck 28 | Dimitri Krassovski @labria * 29 | ??? @nattochaduke-yahoo 30 | ??? @pydeveloper94 31 | Geet Kumar @gkumar7 * 32 | ??? @scharron 33 | Eric Stevens @MightyE * 34 | Rui Chen @oldsharp 35 | ??? @ezacks-barracuda 36 | Tetsuya Morimoto @t2y 37 | Casey Marshall @csmlyve 38 | ??? @jillichrome 39 | ??? @ekooiker * 40 | Chris Bargren @cbargren * 41 | Xian Yi Teng @xytxytxyt * 42 | Stefano Ortolani @ostefano * 43 | Cormoran @cormoran * 44 | ??? @hatokani2 45 | ??? @jeremycnf 46 | Samuel Roberts @sproberts92 * 47 | Alexei Maridashvili @lexigen * 48 | Michael Hamm @MichaelHamm * 49 | Lerh Low @juiceblender * 50 | Peter Palaga @ppalaga * 51 | Hannu Kröger @hkroger * 52 | Jacob Fenwick @jfenwick 53 | Ulises Cervino Beresi @ulises * 54 | Jack Kingsman @jkingsman * 55 | Yasuharu Goto @matope 56 | David Sauer @damoon 57 | -------------------------------------------------------------------------------- /ccmlib/cluster_factory.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | 19 | from __future__ import absolute_import 20 | 21 | import os 22 | 23 | import yaml 24 | 25 | from ccmlib import common, extension, repository 26 | from ccmlib.node import Node 27 | 28 | from distutils.version import LooseVersion #pylint: disable=import-error, no-name-in-module 29 | 30 | class ClusterFactory(): 31 | 32 | @staticmethod 33 | def load(path, name): 34 | cluster_path = os.path.join(path, name) 35 | filename = os.path.join(cluster_path, 'cluster.conf') 36 | with open(filename, 'r') as f: 37 | data = yaml.safe_load(f) 38 | try: 39 | install_dir = None 40 | if 'install_dir' in data: 41 | install_dir = data['install_dir'] 42 | repository.validate(install_dir) 43 | if install_dir is None and 'cassandra_dir' in data: 44 | install_dir = data['cassandra_dir'] 45 | repository.validate(install_dir) 46 | 47 | cassandra_version = None 48 | if 'cassandra_version' in data: 49 | cassandra_version = LooseVersion(data['cassandra_version']) 50 | cluster_class = extension.get_cluster_class(install_dir) 51 | cluster = cluster_class(path, data['name'], install_dir=install_dir, create_directory=False, derived_cassandra_version=cassandra_version) 52 | node_list = data['nodes'] 53 | seed_list = data['seeds'] 54 | if 'partitioner' in data: 55 | cluster.partitioner = data['partitioner'] 56 | if 'config_options' in data: 57 | cluster._config_options = data['config_options'] 58 | if 'dse_config_options' in data: 59 | cluster._dse_config_options = data['dse_config_options'] 60 | if 'misc_config_options' in data: 61 | cluster._misc_config_options = data['misc_config_options'] 62 | if 'log_level' in data: 63 | cluster.__log_level = data['log_level'] 64 | if 'use_vnodes' in data: 65 | cluster.use_vnodes = data['use_vnodes'] 66 | if 'configuration_yaml' in data: 67 | cluster.configuration_yaml = data['configuration_yaml'] 68 | if 'datadirs' in data: 69 | cluster.data_dir_count = int(data['datadirs']) 70 | extension.load_from_cluster_config(cluster, data) 71 | except KeyError as k: 72 | raise common.LoadError("Error Loading " + filename + ", missing property:" + k) 73 | 74 | for node_name in node_list: 75 | cluster.nodes[node_name] = Node.load(cluster_path, node_name, cluster) 76 | for seed in seed_list: 77 | cluster.seeds.append(seed) 78 | 79 | return cluster 80 | -------------------------------------------------------------------------------- /misc/ccm-completion.bash: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ############################################################################## 18 | # ccm-completion.bash 19 | # 20 | # CCM command completion logic for bash. It dynamically determines available 21 | # sub-commands based on the ccm being invoked. Thus, users running multiple 22 | # ccm's (or a ccm that they are continuously updating with new commands) will 23 | # automagically work. 24 | # 25 | # This completion script relies on ccm having two hidden subcommands: 26 | # show-cluster-cmds - emits the names of cluster sub-commands. 27 | # show-node-cmds - emits the names of node sub-commands. 28 | # 29 | # Usage: 30 | # * Copy the script into your home directory or some such. 31 | # * Source it from your ~/.bash_profile: . ~/scripts/ccm-completion.bash 32 | ############################################################################## 33 | 34 | case "$COMP_WORDBREAKS" in 35 | *:*) : great ;; 36 | *) COMP_WORDBREAKS="$COMP_WORDBREAKS:" 37 | esac 38 | 39 | __ccmcomp_is_node () 40 | { 41 | local CANDIDATE=$1 42 | 43 | # Get the list of nodes from 'ccm list' and then see if the given node 44 | # name is in it. 45 | local RC=1 46 | for i in $(__ccmcomp_node_list) ; do 47 | if [ "$CANDIDATE" == "$i" ] ; then 48 | RC=0 49 | break 50 | fi 51 | done 52 | 53 | return $RC 54 | } 55 | 56 | __ccmcomp_node_list () 57 | { 58 | ${COMP_WORDS[0]} status | grep -E ': (DOWN|UP)' | sed -e 's/:.*//' 59 | } 60 | 61 | __ccm_switch () 62 | { 63 | local CMD=${COMP_WORDS[0]} 64 | local WORD=$1 65 | COMPREPLY=( $(compgen -W "$($CMD list | sed -E -e 's/ *\*?//')" $WORD) ) 66 | } 67 | 68 | __ccmcomp_cluster_cmd_filter () 69 | { 70 | local CMD=${COMP_WORDS[0]} 71 | local WORD=$2 72 | local PREV_WORD=$3 73 | # Is this the first arg to ccm? If so, we want cluster commands. 74 | if [ $COMP_CWORD == 1 ] ; then 75 | # We get the cluster-cmds via a background process to make this go a 76 | # bit faster. 10s-100s of milliseconds make a difference here because 77 | # a user is waiting on the completion response. 78 | exec 3< <($CMD show-cluster-cmds) 79 | COMPREPLY=( $(compgen -W "$(__ccmcomp_node_list) $(cat <&3)" $WORD) ) 80 | else 81 | # PREV_WORD is a sub-command or node name. If it's a node name, 82 | # show (filtered) node commands. 83 | if __ccmcomp_is_node $PREV_WORD ; then 84 | COMPREPLY=( $(compgen -W "$($CMD show-node-cmds)" $WORD) ) 85 | else 86 | # It's a subcommand. Call the argument filter function for that 87 | # sub-command if it exists 88 | if type __ccm_$PREV_WORD > /dev/null 2>&1 ; then 89 | __ccm_$PREV_WORD $WORD 90 | else 91 | return 1 92 | fi 93 | fi 94 | fi 95 | } 96 | 97 | # Bind completions for ccm to invoke our top-level handling function. 98 | complete -F __ccmcomp_cluster_cmd_filter ccm 99 | -------------------------------------------------------------------------------- /.github/workflows/main-python-2-7.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | name: CI - Python 2.7 18 | 19 | on: 20 | push: 21 | pull_request: 22 | workflow_dispatch: 23 | 24 | jobs: 25 | lint_test_smoke: 26 | runs-on: ubuntu-24.04 27 | container: 28 | image: python:2.7-slim 29 | 30 | steps: 31 | - uses: actions/checkout@v4 32 | 33 | - name: Install system dependencies 34 | run: | 35 | # Update sources to use Debian archive since Buster is EOL 36 | sed -i 's/deb.debian.org/archive.debian.org/g' /etc/apt/sources.list 37 | sed -i '/security.debian.org/d' /etc/apt/sources.list 38 | sed -i '/stretch-updates/d' /etc/apt/sources.list 39 | sed -i '/buster-updates/d' /etc/apt/sources.list 40 | mkdir -p /usr/share/man/man1 41 | apt-get update 42 | apt-get install -y curl netcat-openbsd sudo git gcc python-dev openjdk-11-jdk-headless ant 43 | 44 | - name: Set up Python environment 45 | run: | 46 | python -m pip install --upgrade 'pip<21.0' 'setuptools<45' 'wheel<0.35' 47 | mkdir -p /github/home/.cache/pip 48 | pip install -r requirements.txt 49 | pip install 'pylint==1.9.5' 50 | 51 | - name: Run linter 52 | run: | 53 | pylint --output-format msvs --reports y ccmlib || true 54 | pylint --output-format msvs --reports y tests || true 55 | 56 | - name: Smoke tests 57 | shell: bash 58 | run: | 59 | export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 60 | # hack to fix setup.cfg not being parsed by pbr (FIXME why isn't pbr working...) 61 | export PBR_VERSION=$(grep version setup.cfg | awk -F= '{print $2}' | xargs) 62 | ./setup.py install 63 | set -x 64 | LATEST_4_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oE '4\.0\.[0-9]+' | sort -V | tail -1) 65 | LATEST_4_1=$(curl -s https://downloads.apache.org/cassandra/ | grep -oE '4\.1\.[0-9]+' | sort -V | tail -1) 66 | ccm_test() { 67 | for i in {1..9}; do 68 | echo "Checking nc -z 127.0.0.1 7000" 69 | while nc -z 127.0.0.1 7000 ; do echo . ; ./ccm stop || true ; sleep 1 ; done 70 | ./ccm start -v --root && ./ccm remove && return 0 || echo retrying 71 | sleep 20 72 | done 73 | echo "ccm start failed after 9 attempts" 74 | exit 1 75 | } 76 | export -f ccm_test 77 | ./ccm create -h 78 | ./ccm create test -v ${LATEST_4_0} -n1 --vnodes --quiet 79 | ccm_test 80 | ./ccm create test -v ${LATEST_4_1} -n1 --vnodes --quiet 81 | ccm_test 82 | ./ccm create test --version='git:cassandra-4.0' -n1 --vnodes --quiet 83 | ccm_test 84 | ./ccm create test --version='git:cassandra-4.1' -n1 --vnodes --quiet 85 | ccm_test 86 | 87 | - name: Publish Test Report 88 | uses: mikepenz/action-junit-report@v5 89 | if: always() 90 | with: 91 | report_paths: 'junit.xml' 92 | annotate_only: true 93 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | name: CI - Python 3.x 18 | 19 | on: 20 | push: 21 | pull_request: 22 | workflow_dispatch: 23 | 24 | jobs: 25 | lint_test_smoke: 26 | runs-on: ubuntu-24.04 27 | strategy: 28 | matrix: 29 | python-version: ['3.8', '3.11'] # TODO add '3.12' '3.13' '3.14' 30 | fail-fast: false 31 | 32 | steps: 33 | - uses: actions/checkout@v4 34 | 35 | - name: Set up Python ${{ matrix.python-version }} 36 | uses: actions/setup-python@v5 37 | with: 38 | python-version: ${{ matrix.python-version }} 39 | 40 | - name: Set up Python environment 41 | run: | 42 | pip install --upgrade pip 43 | pip install -r requirements.txt && pip install -r tests/requirements.txt 44 | 45 | - name: Run linter 46 | run: | 47 | pylint --output-format msvs --reports y ccmlib || true 48 | pylint --output-format msvs --reports y tests || true 49 | 50 | - name: Run tests 51 | run: | 52 | pytest --junitxml=junit.xml tests 53 | 54 | - name: Smoke tests 55 | run: | 56 | sudo ./setup.py install 57 | set -x 58 | LATEST_4_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oE '4\.0\.[0-9]+' | sort -V | tail -1) 59 | LATEST_4_1=$(curl -s https://downloads.apache.org/cassandra/ | grep -oE '4\.1\.[0-9]+' | sort -V | tail -1) 60 | LATEST_5_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oE '5\.0\.[0-9]+' | sort -V | tail -1) 61 | ccm_test() { 62 | for i in {1..9}; do 63 | echo "Checking nc -z 127.0.0.1 7000" 64 | while nc -z 127.0.0.1 7000 ; do echo . ; ./ccm stop || true ; sleep 1 ; done 65 | ./ccm start -v && ./ccm remove && return 0 || echo retrying 66 | sleep 20 67 | done 68 | echo "ccm start failed after 9 attempts" 69 | exit 1 70 | } 71 | export -f ccm_test 72 | ./ccm create -h 73 | ./ccm create test -v ${LATEST_4_0} -n1 --vnodes --quiet 74 | ccm_test 75 | ./ccm create test -v ${LATEST_4_1} -n1 --vnodes --quiet 76 | ccm_test 77 | ./ccm create test -v ${LATEST_5_0} -n1 --vnodes --quiet 78 | ccm_test 79 | ./ccm create test --version='git:cassandra-4.0' -n1 --vnodes --quiet 80 | ccm_test 81 | ./ccm create test --version='git:cassandra-4.1' -n1 --vnodes --quiet 82 | ccm_test 83 | ./ccm create test --version='git:cassandra-5.0' -n1 --vnodes --quiet 84 | ccm_test 85 | ./ccm create test --version='git:trunk' -n1 --vnodes --quiet 86 | ccm_test 87 | ./ccm create test -v 6.8.54 -n1 --vnodes --dse --quiet 88 | ccm_test 89 | 90 | # todo, when hcd is available 91 | #./ccm create test -v 1.1.0 -n1 --vnodes --hcd --quiet 92 | #ccm_test 93 | 94 | - name: Publish Test Report 95 | uses: mikepenz/action-junit-report@v5 96 | if: always() 97 | with: 98 | report_paths: 'junit.xml' 99 | annotate_only: true 100 | -------------------------------------------------------------------------------- /ccm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | import os 20 | import sys 21 | import warnings 22 | 23 | import pkg_resources 24 | from six import print_ 25 | 26 | from ccmlib import common 27 | from ccmlib.cmds import cluster_cmds, node_cmds 28 | from ccmlib.cmds.common import get_command 29 | from ccmlib.remote import (PARAMIKO_IS_AVAILABLE, execute_ccm_remotely, 30 | get_remote_options, get_remote_usage) 31 | 32 | 33 | def print_subcommand_usage(kind): 34 | for cmd_name in (cluster_cmds if kind.lower() == 'cluster' else node_cmds).commands(): 35 | cmd = get_command(kind, cmd_name) 36 | if not cmd: 37 | print_("Internal error, unknown command {0}".format(cmd_name)) 38 | exit(1) 39 | print_(" {0:14} {1}".format(cmd_name, cmd.description())) 40 | 41 | 42 | def print_global_usage(): 43 | print_("Usage:") 44 | remote_options = "" 45 | if PARAMIKO_IS_AVAILABLE: 46 | remote_options = " [remote_options]" 47 | print_(" ccm" + remote_options + " [options]") 48 | print_(" ccm" + remote_options + " [options]") 49 | if PARAMIKO_IS_AVAILABLE: 50 | print_("") 51 | print_(get_remote_usage()) 52 | print_("") 53 | print_("Where is one of") 54 | print_subcommand_usage('cluster') 55 | print_("") 56 | print_("or is the name of a node of the current cluster and is one of") 57 | print_subcommand_usage('node') 58 | exit(1) 59 | 60 | 61 | for entry_point in pkg_resources.iter_entry_points(group='ccm_extension'): 62 | entry_point.load()() 63 | 64 | common.check_win_requirements() 65 | 66 | if len(sys.argv) <= 1: 67 | print_("Missing arguments") 68 | print_global_usage() 69 | 70 | arg1 = sys.argv[1].lower() 71 | 72 | # show-*-cmds are undocumented commands that emit a list of 73 | # the appropriate type of subcommand. This is used by the bash completion script. 74 | if arg1 == 'show-cluster-cmds': 75 | for cmd_name in cluster_cmds.commands(): 76 | print_(cmd_name) 77 | exit(1) 78 | 79 | if arg1 == 'show-node-cmds': 80 | for cmd_name in node_cmds.commands(): 81 | print_(cmd_name) 82 | exit(1) 83 | 84 | # Attempt a remote execution (only occurs if remote options are valid and ssh_host is set) 85 | # NOTE: An execption may be thrown if there are errors 86 | if PARAMIKO_IS_AVAILABLE: 87 | remote_options, ccm_args = get_remote_options(); 88 | if not remote_options is None and not remote_options.ssh_host is None: 89 | ouput, exit_status = execute_ccm_remotely(remote_options, ccm_args) 90 | sys.exit(exit_status) 91 | 92 | if arg1 in cluster_cmds.commands(): 93 | kind = 'cluster' 94 | cmd = arg1 95 | cmd_args = sys.argv[2:] 96 | else: 97 | if len(sys.argv) <= 2: 98 | print_("Missing arguments") 99 | print_global_usage() 100 | kind = 'node' 101 | node = arg1 102 | cmd = sys.argv[2] 103 | cmd_args = [node] + sys.argv[3:] 104 | 105 | if os.getenv('CASSANDRA_HOME') is not None: 106 | warnings.warn("'CASSANDRA_HOME' is set to be {}".format(os.getenv('CASSANDRA_HOME'))) 107 | 108 | cmd = get_command(kind, cmd) 109 | if not cmd: 110 | print_("Unknown node or command: {0}".format(arg1)) 111 | exit(1) 112 | 113 | parser = cmd.get_parser() 114 | 115 | (options, args) = parser.parse_args(cmd_args) 116 | cmd.validate(parser, options, args) 117 | 118 | cmd.run() 119 | -------------------------------------------------------------------------------- /misc/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # vi: set ft=ruby : 19 | 20 | # Configure defaults and/or allow to be overridden 21 | if ENV.has_key?("CCM") 22 | CCM = ENV["CCM"] 23 | else 24 | CCM = "master" 25 | end 26 | if ENV.has_key?("QUIET") 27 | QUIET = ENV["QUIET"] 28 | else 29 | QUIET = "true" 30 | end 31 | 32 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 33 | VAGRANTFILE_API_VERSION = "2" 34 | 35 | # Inline provision script 36 | CCM_PROVISION_SCRIPT = < ${REDIRECT} 2>&1 54 | # Add JDK repository and update packages 55 | add-apt-repository ppa:webupd8team/java -y > ${REDIRECT} 2>&1 56 | apt-get update -y > ${REDIRECT} 2>&1 57 | apt-get upgrade -y > ${REDIRECT} 2>&1 58 | 59 | # Auto accept the the Java license aggreement 60 | echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections 61 | echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections 62 | 63 | # Install the packages 64 | apt-get install -qq ant git libxml2-dev libxslt1-dev libyaml-dev maven oracle-java8-installer \\ 65 | oracle-java8-unlimited-jce-policy python2.7-dev python-pip vim-gtk \\ 66 | zlib1g-dev > ${REDIRECT} 2>&1 67 | 68 | # Upgrade pip 69 | pip install --upgrade pip > ${REDIRECT} 2>&1 70 | 71 | # Install CCM and its dependencies 72 | echo Installing CCM ... 73 | pip install git+https://github.com/pcmanus/ccm.git@${CCM_VERSION} > ${REDIRECT} 2>&1 74 | EOF 75 | 76 | ## 77 | # Configure a 6 node Cassandra Cluster Manager (CCM) Virtual Machine (VM) with 78 | # the following settings: 79 | # 80 | # - 4GB of RAM 81 | # - 32MB of Video RAM 82 | # - 4 cores (CPUs) 83 | # - Hostname: ccm-cluster 84 | # - Username: vagrant 85 | # - Password: vagrant 86 | # - 6 Network Interfaces Cards (NICs) 87 | # - Node 1: 192.168.33.11 88 | # - Node 2: 192.168.33.12 89 | # - Node 3: 192.168.33.13 90 | # - Node 4: 192.168.33.14 91 | # - Node 5: 192.168.33.15 92 | # - Node 6: 192.168.33.16 93 | ## 94 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 95 | # Create Ubuntu 16.04 LTS VM 96 | config.vm.box = "bento/ubuntu-16.04" 97 | 98 | # Define the hostname and IP addresses (6 node cluster) 99 | config.vm.define "ccm-cluster" do |ccm_cluster| 100 | ccm_cluster.vm.hostname = "ccm-cluster" 101 | ccm_cluster.vm.network "private_network", ip: "192.168.33.11" 102 | ccm_cluster.vm.network "private_network", ip: "192.168.33.12" 103 | ccm_cluster.vm.network "private_network", ip: "192.168.33.13" 104 | ccm_cluster.vm.network "private_network", ip: "192.168.33.14" 105 | ccm_cluster.vm.network "private_network", ip: "192.168.33.15" 106 | ccm_cluster.vm.network "private_network", ip: "192.168.33.16" 107 | end 108 | 109 | # Prepare/Provision the VM 110 | config.vm.provision :shell do |root_provision| 111 | root_provision.privileged = true 112 | root_provision.inline = CCM_PROVISION_SCRIPT 113 | root_provision.args = [ "#{CCM}", "#{QUIET}" ] 114 | end 115 | 116 | # VM parameters for the CCM cluster 117 | config.vm.provider :virtualbox do |provider| 118 | provider.name = "ccm-cluster" 119 | provider.customize ["modifyvm", :id, "--groups", "/Testing"] 120 | provider.customize ["modifyvm", :id, "--memory", "4096"] 121 | provider.customize ["modifyvm", :id, "--vram", "32"] 122 | provider.customize ["modifyvm", :id, "--cpus", "4"] 123 | provider.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] 124 | provider.customize ["modifyvm", :id, "--natdnsproxy1", "on"] 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /tests/test_common.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import unittest 19 | from mock import patch 20 | 21 | from distutils.version import LooseVersion 22 | 23 | from ccmlib import common 24 | from . import ccmtest 25 | 26 | 27 | class TestCommon(ccmtest.Tester): 28 | 29 | def test_normalize_interface(self): 30 | normalized = common.normalize_interface(('::1', 9042)) 31 | self.assertEqual(normalized, ('0:0:0:0:0:0:0:1', 9042)) 32 | 33 | normalized = common.normalize_interface(('127.0.0.1', 9042)) 34 | self.assertEqual(normalized, ('127.0.0.1', 9042)) 35 | 36 | normalized = common.normalize_interface(('fe80::3e15:c2ff:fed3:db74%en0', 9042)) 37 | self.assertEqual(normalized, ('fe80:0:0:0:3e15:c2ff:fed3:db74%en0', 9042)) 38 | 39 | normalized = common.normalize_interface(('fe80::1%lo0', 9042)) 40 | self.assertEqual(normalized, ('fe80:0:0:0:0:0:0:1%lo0', 9042)) 41 | 42 | normalized = common.normalize_interface(('fd6d:404d:54cb::1', 9042)) 43 | self.assertEqual(normalized, ('fd6d:404d:54cb:0:0:0:0:1', 9042)) 44 | 45 | @patch('ccmlib.common.is_win') 46 | def test_is_modern_windows_install(self, mock_is_win): 47 | mock_is_win.return_value = True 48 | self.assertTrue(common.is_modern_windows_install(2.1)) 49 | self.assertTrue(common.is_modern_windows_install('2.1')) 50 | self.assertTrue(common.is_modern_windows_install(LooseVersion('2.1'))) 51 | 52 | self.assertTrue(common.is_modern_windows_install(3.12)) 53 | self.assertTrue(common.is_modern_windows_install('3.12')) 54 | self.assertTrue(common.is_modern_windows_install(LooseVersion('3.12'))) 55 | 56 | self.assertFalse(common.is_modern_windows_install(1.0)) 57 | self.assertFalse(common.is_modern_windows_install('1.0')) 58 | self.assertFalse(common.is_modern_windows_install(LooseVersion('1.0'))) 59 | 60 | def test_merge_configuration(self): 61 | # test for merging dict val in key, value pair 62 | dict0 = dict1 = {'key': {'val1': True}} 63 | dict2 = {'key': {'val2': False}} 64 | 65 | for k, v in dict2.items(): 66 | dict0[k].update(v) 67 | 68 | self.assertEqual(common.merge_configuration(dict1, dict2), dict0) 69 | 70 | # test for merging str val in key, value pair 71 | dict0 = dict1 = {'key': 'val1'} 72 | dict2 = {'key': 'val2'} 73 | 74 | for k, v in dict2.items(): 75 | dict0[k] = v 76 | 77 | self.assertEqual(common.merge_configuration(dict1, dict2), dict0) 78 | 79 | def test_get_jdk_version(self): 80 | v8u152 = """java version "1.8.0_152" 81 | Java(TM) SE Runtime Environment (build 1.8.0_152-b16) 82 | Java HotSpot(TM) 64-Bit Server VM (build 25.152-b16, mixed mode) 83 | """ 84 | # Since Java 9, the version string syntax changed. 85 | # Most relevant change is that trailing .0's are omitted. I.e. Java "9.0.0" 86 | # version string is not "9.0.0" but just "9". 87 | v900 = """java version "9" 88 | Java(TM) SE Runtime Environment (build 9+1) 89 | Java HotSpot(TM) 64-Bit Server VM (build 9+1, mixed mode) 90 | """ 91 | v901 = """java version "9.0.1" 92 | Java(TM) SE Runtime Environment (build 9.0.1+11) 93 | Java HotSpot(TM) 64-Bit Server VM (build 9.0.1+11, mixed mode) 94 | """ 95 | # 10-internal, just to have an internal (local) build in here 96 | v10_int = """openjdk version "10-internal" 97 | OpenJDK Runtime Environment (build 10-internal+0-adhoc.jenkins.openjdk-shenandoah-jdk10-release) 98 | OpenJDK 64-Bit Server VM (build 10-internal+0-adhoc.jenkins.openjdk-shenandoah-jdk10-release, mixed mode) 99 | """ 100 | v1000 = """java version "10" 101 | Java(TM) SE Runtime Environment (build 9+1) 102 | Java HotSpot(TM) 64-Bit Server VM (build 9+1, mixed mode) 103 | """ 104 | v1001 = """java version "10.0.1" 105 | Java(TM) SE Runtime Environment (build 10.0.1+11) 106 | Java HotSpot(TM) 64-Bit Server VM (build 10.0.1+11, mixed mode) 107 | """ 108 | 109 | self.assertEqual(common._get_jdk_version(v8u152), "1.8") 110 | self.assertEqual(common._get_jdk_version(v900), "9.0") 111 | self.assertEqual(common._get_jdk_version(v901), "9.0") 112 | self.assertEqual(common._get_jdk_version(v10_int), "10.0") 113 | self.assertEqual(common._get_jdk_version(v1000), "10.0") 114 | self.assertEqual(common._get_jdk_version(v1001), "10.0") 115 | 116 | if __name__ == '__main__': 117 | unittest.main() 118 | -------------------------------------------------------------------------------- /ccmlib/cmds/command.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | 19 | from __future__ import absolute_import 20 | 21 | import os 22 | import sys 23 | from optparse import BadOptionError, Option, OptionParser 24 | import re 25 | 26 | from six import print_ 27 | 28 | import ccmlib 29 | from ccmlib import common 30 | from ccmlib.cluster_factory import ClusterFactory 31 | from ccmlib.remote import PARAMIKO_IS_AVAILABLE, get_remote_usage 32 | 33 | 34 | # This is fairly fragile, but handy for now 35 | class ForgivingParser(OptionParser): 36 | 37 | def __init__(self, usage=None, option_list=None, option_class=Option, version=None, conflict_handler="error", description=None, formatter=None, add_help_option=True, prog=None, epilog=None): 38 | OptionParser.__init__(self, usage, option_list, option_class, version, conflict_handler, description, formatter, add_help_option, prog, epilog) 39 | self.ignored = [] 40 | 41 | def _process_short_opts(self, rargs, values): 42 | opt = rargs[0] 43 | try: 44 | OptionParser._process_short_opts(self, rargs, values) 45 | except BadOptionError: 46 | self.ignored.append(opt) 47 | self.eat_args(rargs) 48 | 49 | def _process_long_opt(self, rargs, values): 50 | opt = rargs[0] 51 | try: 52 | OptionParser._process_long_opt(self, rargs, values) 53 | except BadOptionError: 54 | self.ignored.append(opt) 55 | self.eat_args(rargs) 56 | 57 | def eat_args(self, rargs): 58 | while len(rargs) > 0 and rargs[0][0] != '-': 59 | self.ignored.append(rargs.pop(0)) 60 | 61 | def get_ignored(self): 62 | return self.ignored 63 | 64 | 65 | class Cmd(object): 66 | options_list = [] 67 | usage = "" 68 | descr_text = "" 69 | ignore_unknown_options = False 70 | 71 | def get_parser(self): 72 | if self.usage == "": 73 | pass 74 | # Do not collapse to PARAMIKO_IS_AVAILABLE, we need to pull it dynamically for testing purposes 75 | if ccmlib.remote.PARAMIKO_IS_AVAILABLE: 76 | self.usage = self.usage.replace("usage: ccm", 77 | "usage: ccm [remote_options]") + \ 78 | os.linesep + os.linesep + \ 79 | get_remote_usage() 80 | parser = self._get_default_parser(self.usage, self.description(), self.ignore_unknown_options) 81 | for args, kwargs in self.options_list: 82 | parser.add_option(*args, **kwargs) 83 | return parser 84 | 85 | def validate(self, parser, options, args, cluster_name=False, node_name=False, load_cluster=False, load_node=True): 86 | self.options = options 87 | self.args = args 88 | if options.config_dir is None: 89 | self.path = common.get_default_path() 90 | else: 91 | self.path = options.config_dir 92 | 93 | if cluster_name: 94 | if len(args) == 0: 95 | print_('Missing cluster name', file=sys.stderr) 96 | parser.print_help() 97 | exit(1) 98 | if not re.match('^[a-zA-Z0-9_-]+$', args[0]): 99 | print_('Cluster name should only contain word characters or hyphen', file=sys.stderr) 100 | exit(1) 101 | self.name = args[0] 102 | if node_name: 103 | if len(args) == 0: 104 | print_('Missing node name', file=sys.stderr) 105 | parser.print_help() 106 | exit(1) 107 | self.name = args[0] 108 | 109 | if load_cluster: 110 | self.cluster = self._load_current_cluster() 111 | if node_name and load_node: 112 | try: 113 | self.node = self.cluster.nodes[self.name] 114 | except KeyError: 115 | print_('Unknown node %s in cluster %s' % (self.name, self.cluster.name), file=sys.stderr) 116 | exit(1) 117 | 118 | def run(self): 119 | pass 120 | 121 | def _get_default_parser(self, usage, description, ignore_unknown_options=False): 122 | if ignore_unknown_options: 123 | parser = ForgivingParser(usage=usage, description=description) 124 | else: 125 | parser = OptionParser(usage=usage, description=description) 126 | parser.add_option('--config-dir', type="string", dest="config_dir", 127 | help="Directory for the cluster files [default to {0}]".format(common.get_default_path_display_name())) 128 | return parser 129 | 130 | def description(self): 131 | return self.descr_text 132 | 133 | def _load_current_cluster(self): 134 | name = common.current_cluster_name(self.path) 135 | if name is None: 136 | print_('No currently active cluster (use ccm cluster switch)') 137 | exit(1) 138 | try: 139 | return ClusterFactory.load(self.path, name) 140 | except common.LoadError as e: 141 | print_(str(e)) 142 | exit(1) 143 | -------------------------------------------------------------------------------- /ccmlib/hcd/hcd_cluster.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # DataStax Hyper-Converged Database (HCD) clusters 18 | 19 | from __future__ import absolute_import 20 | 21 | import os 22 | import re 23 | import subprocess 24 | import shutil 25 | import tarfile 26 | import tempfile 27 | from argparse import ArgumentError 28 | from distutils.version import LooseVersion 29 | 30 | from ccmlib import common, repository 31 | from ccmlib.cluster import Cluster 32 | from ccmlib.common import ArgumentError, rmdirs 33 | from ccmlib.hcd.hcd_node import HcdNode 34 | 35 | try: 36 | import ConfigParser 37 | except ImportError: 38 | import configparser as ConfigParser 39 | 40 | 41 | HCD_CASSANDRA_CONF_DIR = "resources/cassandra/conf" 42 | HCD_ARCHIVE = "https://downloads.datastax.com/hcd/hcd-%s-bin.tar.gz" 43 | 44 | 45 | def isHcd(install_dir, options=None): 46 | if install_dir is None: 47 | raise ArgumentError('Undefined installation directory') 48 | bin_dir = os.path.join(install_dir, common.BIN_DIR) 49 | if options and options.hcd and './' != install_dir and not os.path.exists(bin_dir): 50 | raise ArgumentError('Installation directory does not contain a bin directory: %s' % install_dir) 51 | if options and options.hcd: 52 | return True 53 | hcd_script = os.path.join(bin_dir, 'hcd') 54 | if options and not options.hcd and './' != install_dir and os.path.exists(hcd_script): 55 | raise ArgumentError('Installation directory is HCD but options did not specify `--hcd`: %s' % install_dir) 56 | 57 | return os.path.exists(hcd_script) 58 | 59 | def isHcdClusterType(install_dir, options=None): 60 | if isHcd(install_dir, options): 61 | return HcdCluster 62 | return None 63 | 64 | 65 | class HcdCluster(Cluster): 66 | 67 | @staticmethod 68 | def getConfDir(install_dir): 69 | if isHcd(install_dir): 70 | return os.path.join(install_dir, HCD_CASSANDRA_CONF_DIR) 71 | raise RuntimeError("illegal call to HcdCluster.getConfDir() when not HCD") 72 | 73 | @staticmethod 74 | def getNodeClass(): 75 | return HcdNode 76 | 77 | 78 | def __init__(self, path, name, partitioner=None, install_dir=None, create_directory=True, version=None, verbose=False, derived_cassandra_version=None, options=None): 79 | self._cassandra_version = None 80 | if derived_cassandra_version: 81 | self._cassandra_version = derived_cassandra_version 82 | 83 | super(HcdCluster, self).__init__(path, name, partitioner, install_dir, create_directory, version, verbose, options) 84 | 85 | def can_generate_tokens(self): 86 | return False 87 | 88 | def load_from_repository(self, version, verbose): 89 | return setup_hcd(version, verbose) 90 | 91 | def create_node(self, name, auto_bootstrap, thrift_interface, storage_interface, jmx_port, remote_debug_port, initial_token, save=True, binary_interface=None, byteman_port='0', environment_variables=None, derived_cassandra_version=None, **kwargs): 92 | return HcdNode(name, self, auto_bootstrap, thrift_interface, storage_interface, jmx_port, remote_debug_port, initial_token, save, binary_interface, byteman_port, environment_variables=environment_variables, derived_cassandra_version=derived_cassandra_version, **kwargs) 93 | 94 | def cassandra_version(self): 95 | if self._cassandra_version is None: 96 | self._cassandra_version = get_hcd_cassandra_version(self.get_install_dir()) 97 | return self._cassandra_version 98 | 99 | 100 | def download_hcd_version(version, verbose=False): 101 | url = HCD_ARCHIVE 102 | if repository.CCM_CONFIG.has_option('repositories', 'hcd'): 103 | url = repository.CCM_CONFIG.get('repositories', 'hcd') 104 | 105 | url = url % version 106 | _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") 107 | try: 108 | repository.__download(url, target, show_progress=verbose) 109 | common.debug("Extracting {} as version {} ...".format(target, version)) 110 | tar = tarfile.open(target) 111 | dir = tar.next().name.split("/")[0] # pylint: disable=all 112 | tar.extractall(path=repository.__get_dir()) 113 | tar.close() 114 | target_dir = os.path.join(repository.__get_dir(), version) 115 | if os.path.exists(target_dir): 116 | rmdirs(target_dir) 117 | shutil.move(os.path.join(repository.__get_dir(), dir), target_dir) 118 | except urllib.error.URLError as e: 119 | msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url 120 | msg = msg + " (underlying error is: %s)" % str(e) 121 | raise ArgumentError(msg) 122 | except tarfile.ReadError as e: 123 | raise ArgumentError("Unable to uncompress downloaded file: %s" % str(e)) 124 | 125 | 126 | def setup_hcd(version, verbose=False): 127 | (cdir, version, fallback) = repository.__setup(version, verbose) 128 | if cdir: 129 | return (cdir, version) 130 | cdir = repository.version_directory(version) 131 | if cdir is None: 132 | download_hcd_version(version, verbose=verbose) 133 | cdir = repository.version_directory(version) 134 | return (cdir, version) 135 | 136 | 137 | def get_hcd_version(install_dir): 138 | """ look for a hcd*.jar and extract the version number """ 139 | for root, dirs, files in os.walk(install_dir): 140 | for file in files: 141 | match = re.search('^hcd(?:-core)?-([0-9.]+)(?:-.*)?\.jar', file) 142 | if match: 143 | return match.group(1) 144 | return None 145 | 146 | 147 | def get_hcd_cassandra_version(install_dir): 148 | # for this to work, the current JAVA_HOME must already be appropriate 149 | hcd_cmd = os.path.join(install_dir, 'bin', 'hcd') 150 | (output, stderr) = subprocess.Popen([hcd_cmd, "cassandra", '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() 151 | # just take the last line to avoid any possible logback log lines 152 | output = output.decode('utf-8').rstrip().split('\n')[-1] 153 | match = re.search('([0-9.]+)(?:-.*)?', str(output)) 154 | if match: 155 | return LooseVersion(match.group(1)) 156 | raise ArgumentError("Unable to determine Cassandra version in: %s.\n\tstdout: '%s'\n\tstderr: '%s'" 157 | % (install_dir, output, stderr)) 158 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | CCM (Cassandra Cluster Manager) 2 | ==================================================== 3 | 4 | 5 | New to Python development? 6 | -------------------------- 7 | Python has moved on since CCM started development. `pip` is the new `easy_install`, 8 | Python 3 is the new 2.7, and pyenv and virtualenv are strongly recommended for managing 9 | multiple Python versions and dependencies for specific Python applications. 10 | 11 | A typical macOS setup would be to install [Homebrew](https://docs.brew.sh/Installation), 12 | then `brew install pyenv` to manage Python versions and then use virtualenv to 13 | manage the dependencies for CCM. Make sure to add [brew's bin directory to your path in 14 | your ~/.zshenv](https://www.zerotohero.dev/zshell-startup-files/). This would be 15 | `/usr/local` for macOS Intel and `/opt/homebrew/` for macOS on Apple Silicon. 16 | 17 | Now you are ready to install Python using pyenv. To avoid getting a bleeding edge version that will fail with 18 | some aspect of CCM you can `pyenv install 3.9.16`. 19 | 20 | To create the virtualenv run `python3 -m venv --prompt ccm venv` with your git repo as the 21 | current working directory to create a virtual environment for CCM. Then `source venv/bin/activate` to 22 | enable the venv for the current terminal and `deactivate` to exit. 23 | 24 | Now you a ready to set up the venv with CCM and its test dependencies. `pip install -e ` 25 | to install CCM, and its runtime dependencies from `requirements.txt`, so that the version of 26 | CCM you are running points to the code you are actively working on. There is no build or package step because you 27 | are editing the Python files being run every time you invoke CCM. 28 | 29 | Almost there. Now you just need to add the test dependencies that are not in `requirements.txt`. 30 | `pip install mock pytest requests` to finish setting up your dev environment! 31 | 32 | Another caveat that has recently appeared in Cassandra versions 4.0 and below is they all ship with a version of JNA that isn't 33 | compatible with Apple Silicon and there are no plans to update JNA on those versions. One work around if you are 34 | generally building Cassandra from source to use with CCM is to replace the JNA jar in your Maven repo with a [newer 35 | one](https://search.maven.org/artifact/net.java.dev.jna/jna/5.8.0/jar) that supports Apple Silicon. 36 | Which you version you need to replace will vary depending on the Cassandra version, but it will normally be in 37 | `~/.m2/repository/net/java/dev/jna/jna/`. You can also replace the library in 38 | `~/.ccm/repository//lib`. 39 | 40 | Also don't forget to disable `AirPlay Receiver` on macOS which also listens on port `7000`. 41 | 42 | Requirements 43 | ------------ 44 | 45 | - A working python installation (tested to work with python 2.7). 46 | - See `requirements.txt` for runtime requirements 47 | - `mock` and `pytest` for tests 48 | - ant (http://ant.apache.org/, on macOS X, `brew install ant`) 49 | - Java: Cassandra currently builds with either 8 or 11 and is restricted to JDK 8 language 50 | features and dependencies. There are several sources for the JDK and Azul Zulu is one good option. 51 | - If you want to create multiple node clusters, the simplest way is to use 52 | multiple loopback aliases. On modern linux distributions you probably don't 53 | need to do anything, but on macOS X, you will need to create the aliases with 54 | 55 | sudo ifconfig lo0 alias 127.0.0.2 up 56 | sudo ifconfig lo0 alias 127.0.0.3 up 57 | ... 58 | 59 | Note that the usage section assumes that at least 127.0.0.1, 127.0.0.2 and 60 | 127.0.0.3 are available. 61 | 62 | ### Optional Requirements 63 | 64 | - Paramiko (http://www.paramiko.org/): Paramiko adds the ability to execute CCM 65 | remotely; `pip install paramiko` 66 | 67 | __Note__: The remote machine must be configured with an SSH server and a working 68 | CCM. When working with multiple nodes each exposed IP address must be 69 | in sequential order. For example, the last number in the 4th octet of 70 | a IPv4 address must start with `1` (e.g. 192.168.33.11). See 71 | [Vagrantfile](misc/Vagrantfile) for help with configuration of remote 72 | CCM machine. 73 | 74 | 75 | Known issues 76 | ------------ 77 | Windows only: 78 | - `node start` pops up a window, stealing focus. 79 | - cqlsh started from ccm show incorrect prompts on command-prompt 80 | - non nodetool-based command-line options fail (sstablesplit, scrub, etc) 81 | - To install psutil, you must use the .msi from pypi. pip install psutil will not work 82 | - You will need ant.bat in your PATH in order to build C* from source 83 | - You must run with an Unrestricted Powershell Execution-Policy if using Cassandra 2.1.0+ 84 | - Ant installed via [chocolatey](https://chocolatey.org/) will not be found by ccm, so you must create a symbolic 85 | link in order to fix the issue (as administrator): 86 | - cmd /c mklink C:\ProgramData\chocolatey\bin\ant.bat C:\ProgramData\chocolatey\bin\ant.exe 87 | 88 | macOS only: 89 | - Airplay listens for incoming connections on 7000 so disable `Settings` -> `General` -> `AirDrop & Handoff` -> `AirPlay Receiver` 90 | 91 | Remote Execution only: 92 | - Using `--config-dir` and `--install-dir` with `create` may not work as 93 | expected; since the configuration directory and the installation directory 94 | contain lots of files they will not be copied over to the remote machine 95 | like most other options for cluster and node operations 96 | - cqlsh started from ccm using remote execution will not start 97 | properly (e.g.`ccm --ssh-host 192.168.33.11 node1 cqlsh`); however 98 | `-x ` or `--exec=CMDS` can still be used to execute a CQLSH command 99 | on a remote node. 100 | 101 | Installation 102 | ------------ 103 | 104 | ccm uses python distutils so from the source directory run: 105 | 106 | sudo ./setup.py install 107 | 108 | ccm is available on the [Python Package Index][pip]: 109 | 110 | pip install ccm 111 | 112 | There is also a [Homebrew package][brew] available: 113 | 114 | brew install ccm 115 | 116 | [pip]: https://pypi.org/project/ccm/ 117 | [brew]: https://github.com/Homebrew/homebrew-core/blob/master/Formula/ccm.rb 118 | 119 | 120 | Testing 121 | ----------------------- 122 | 123 | Create a virtual environment i.e.: 124 | 125 | python3 -m venv ccm 126 | 127 | `pip install` all dependencies as well as `mock` and `pytest`. Run `pytest` from the repository root to run the tests. 128 | 129 | 130 | CCM Lib 131 | ------- 132 | 133 | The ccm facilities are available programmatically through ccmlib. This could 134 | be used to implement automated tests against Cassandra. A simple example of 135 | how to use ccmlib follows: 136 | 137 | import ccmlib.cluster 138 | 139 | CLUSTER_PATH="." 140 | cluster = ccmlib.cluster.Cluster(CLUSTER_PATH, 'test', cassandra_version='2.1.14') 141 | cluster.populate(3).start() 142 | [node1, node2, node3] = cluster.nodelist() 143 | 144 | # do some tests on the cluster/nodes. To connect to a node through thrift, 145 | # the host and port to a node is available through 146 | # node.network_interfaces['thrift'] 147 | 148 | cluster.flush() 149 | node2.compact() 150 | 151 | # do some other tests 152 | 153 | # after the test, you can leave the cluster running, you can stop all nodes 154 | # using cluster.stop() but keep the data around (in CLUSTER_PATH/test), or 155 | # you can remove everything with cluster.remove() 156 | -------------------------------------------------------------------------------- /ccmlib/hcd/hcd_node.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # DataStax Hyper-Converged Database (HCD) nodes 18 | 19 | from __future__ import absolute_import, with_statement 20 | 21 | import os 22 | import shutil 23 | import yaml 24 | 25 | from distutils.version import LooseVersion 26 | from six.moves import urllib, xrange 27 | 28 | from ccmlib import common, node 29 | from ccmlib.node import Node 30 | 31 | 32 | 33 | class HcdNode(Node): 34 | 35 | """ 36 | Provides interactions to a HCD node. 37 | """ 38 | 39 | @staticmethod 40 | def get_version_from_build(install_dir=None, node_path=None, cassandra=False): 41 | if install_dir is None and node_path is not None: 42 | install_dir = node.get_install_dir_from_cluster_conf(node_path) 43 | if install_dir is not None: 44 | # Binary cassandra installs will have a 0.version.txt file 45 | version_file = os.path.join(install_dir, '0.version.txt') 46 | if os.path.exists(version_file): 47 | with open(version_file) as f: 48 | return LooseVersion(f.read().strip()) 49 | # For HCD look for a hcd*.jar and extract the version number 50 | from ccmlib.hcd.hcd_cluster import get_hcd_version 51 | hcd_version = get_hcd_version(install_dir) 52 | if (hcd_version is not None): 53 | if cassandra: 54 | from ccmlib.hcd.hcd_cluster import get_hcd_cassandra_version 55 | return get_hcd_cassandra_version(install_dir) 56 | else: 57 | return LooseVersion(hcd_version) 58 | raise common.CCMError("Cannot find version") 59 | 60 | 61 | def __init__(self, name, cluster, auto_bootstrap, thrift_interface, storage_interface, jmx_port, remote_debug_port, initial_token, save=True, binary_interface=None, byteman_port='0', environment_variables=None, derived_cassandra_version=None, **kwargs): 62 | super(HcdNode, self).__init__(name, cluster, auto_bootstrap, thrift_interface, storage_interface, jmx_port, remote_debug_port, initial_token, save, binary_interface, byteman_port, environment_variables=environment_variables, derived_cassandra_version=derived_cassandra_version, **kwargs) 63 | 64 | def get_install_cassandra_root(self): 65 | return os.path.join(self.get_install_dir(), 'distribution', 'hcd' , 'target', 'hcd', 'resources', 'cassandra') 66 | 67 | def get_node_cassandra_root(self): 68 | return os.path.join(self.get_path(), 'resources', 'cassandra') 69 | 70 | def get_conf_dir(self): 71 | """ 72 | Returns the path to the directory where Cassandra config are located 73 | """ 74 | return os.path.join(self.get_path(), 'resources', 'cassandra', 'conf') 75 | 76 | def get_tool(self, toolname): 77 | return common.join_bin(os.path.join(self.get_install_dir(), 'distribution', 'hcd' , 'target', 'hcd', 'resources', 'cassandra'), 'bin', toolname) 78 | 79 | def get_tool_args(self, toolname): 80 | return [common.join_bin(os.path.join(self.get_install_dir(), 'distribution', 'hcd' , 'target', 'hcd', 'resources', 'cassandra'), 'bin', toolname)] 81 | 82 | def get_env(self): 83 | env = self.make_hcd_env(self.get_install_dir(), self.get_path()) 84 | # adjust JAVA_HOME to one supported by this hcd_version 85 | env = common.update_java_version(jvm_version=None, 86 | install_dir=self.get_install_dir(), 87 | env=env, 88 | info_message=self.name) 89 | return env 90 | 91 | def node_setup(self, version, verbose): 92 | from ccmlib.hcd.hcd_cluster import setup_hcd 93 | dir, v = setup_hcd(version, verbose=verbose) 94 | return dir 95 | 96 | def get_launch_bin(self): 97 | cdir = os.path.join(self.get_install_dir(), 'distribution', 'hcd' , 'target', 'hcd') 98 | launch_bin = common.join_bin(cdir, 'bin', 'hcd') 99 | shutil.copy(launch_bin, self.get_bin_dir()) 100 | return common.join_bin(self.get_path(), 'bin', 'hcd') 101 | 102 | def add_custom_launch_arguments(self, args): 103 | args.append('cassandra') 104 | 105 | def copy_config_files(self): 106 | for product in ['hcd', 'cassandra']: 107 | src_conf = os.path.join(self.get_install_dir(), 'distribution', 'hcd' , 'target', 'hcd', 'resources', product, 'conf') 108 | dst_conf = os.path.join(self.get_path(), 'resources', product, 'conf') 109 | if not os.path.isdir(src_conf): 110 | continue 111 | if os.path.isdir(dst_conf): 112 | common.rmdirs(dst_conf) 113 | shutil.copytree(src_conf, dst_conf) 114 | 115 | def import_bin_files(self): 116 | common.copy_directory(os.path.join(self.get_install_dir(), 'bin'), self.get_bin_dir()) 117 | cassandra_bin_dir = os.path.join(self.get_path(), 'resources', 'cassandra', 'bin') 118 | shutil.rmtree(cassandra_bin_dir, ignore_errors=True) 119 | os.makedirs(cassandra_bin_dir) 120 | common.copy_directory(os.path.join(self.get_install_dir(), 'distribution', 'hcd' , 'target', 'hcd', 'resources', 'cassandra', 'bin'), cassandra_bin_dir) 121 | if os.path.exists(os.path.join(self.get_install_dir(), 'distribution', 'hcd' , 'target', 'hcd', 'resources', 'cassandra', 'tools')): 122 | cassandra_tools_dir = os.path.join(self.get_path(), 'resources', 'cassandra', 'tools') 123 | shutil.rmtree(cassandra_tools_dir, ignore_errors=True) 124 | shutil.copytree(os.path.join(self.get_install_dir(), 'distribution', 'hcd' , 'target', 'hcd', 'resources', 'cassandra', 'tools'), cassandra_tools_dir) 125 | self.export_hcd_home_in_hcd_env_sh() 126 | 127 | 128 | def export_hcd_home_in_hcd_env_sh(self): 129 | ''' 130 | Due to the way CCM lays out files, separating the repository 131 | from the node(s) confs, the `hcd-env.sh` script of each node 132 | needs to have its HCD_HOME var set and exported. 133 | The stock `hcd-env.sh` file includes a commented-out 134 | place to do exactly this, intended for installers. 135 | Basically: read in the file, write it back out and add the two 136 | lines. 137 | 'sstableloader' is an example of a node script that depends on 138 | this, when used in a CCM-built cluster. 139 | ''' 140 | with open(self.get_bin_dir() + "/hcd-env.sh", "r") as hcd_env_sh: 141 | buf = hcd_env_sh.readlines() 142 | 143 | with open(self.get_bin_dir() + "/hcd-env.sh", "w") as out_file: 144 | for line in buf: 145 | out_file.write(line) 146 | if line == "# This is here so the installer can force set HCD_HOME\n": 147 | out_file.write("HCD_HOME=" + os.path.join(self.get_install_dir(), 'distribution', 'hcd' , 'target', 'hcd') + "\nexport HCD_HOME\n") 148 | 149 | 150 | def _get_directories(self): 151 | dirs = [] 152 | for i in ['commitlogs', 'saved_caches', 'logs', 'bin', 'resources']: 153 | dirs.append(os.path.join(self.get_path(), i)) 154 | for x in xrange(0, self.cluster.data_dir_count): 155 | dirs.append(os.path.join(self.get_path(), 'data{0}'.format(x))) 156 | return dirs 157 | 158 | def make_hcd_env(self, install_dir, node_path): 159 | env = os.environ.copy() 160 | env['MAX_HEAP_SIZE'] = os.environ.get('CCM_MAX_HEAP_SIZE', '500M') 161 | env['HEAP_NEWSIZE'] = os.environ.get('CCM_HEAP_NEWSIZE', '50M') 162 | env['MAX_DIRECT_MEMORY'] = os.environ.get('CCM_MAX_DIRECT_SIZE', '2048M') 163 | env['HCD_HOME'] = os.path.join(install_dir, 'distribution', 'hcd' , 'target', 'hcd') 164 | env['HCD_CONF'] = os.path.join(node_path, 'resources', 'hcd', 'conf') 165 | env['CASSANDRA_HOME'] = os.path.join(install_dir, 'distribution', 'hcd' , 'target', 'hcd', 'resources', 'cassandra') 166 | env['CASSANDRA_CONF'] = os.path.join(node_path, 'resources', 'cassandra', 'conf') 167 | env['HCD_LOG_ROOT'] = os.path.join(node_path, 'logs', 'hcd') 168 | env['CASSANDRA_LOG_DIR'] = os.path.join(node_path, 'logs') 169 | return env 170 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 2 | ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/ccm) 3 | [![PyPI version](https://badge.fury.io/py/ccm.svg)](https://badge.fury.io/py/ccm) 4 | 5 | CCM (Cassandra Cluster Manager) 6 | ==================================================== 7 | 8 | 9 | A script/library to create, launch and remove an Apache Cassandra cluster on 10 | localhost. 11 | 12 | The goal of ccm and ccmlib is to make it easy to create, manage and destroy a 13 | small Cassandra cluster on a local box. It is meant for testing a Cassandra cluster. 14 | 15 | 16 | Installation 17 | ------------ 18 | 19 | See [INSTALL.md](./INSTALL.md). 20 | 21 | Usage 22 | ----- 23 | 24 | Let's say you wanted to fire up a 3 node Cassandra cluster. 25 | 26 | ### Short version 27 | 28 | ccm create test -v 2.0.5 -n 3 -s 29 | 30 | You will of course want to replace `2.0.5` by whichever version of Cassandra 31 | you want to test. 32 | 33 | ### Longer version 34 | 35 | ccm works from a Cassandra source tree (not the jars). There are two ways to 36 | tell ccm how to find the sources: 37 | 1. If you have downloaded *and* compiled Cassandra sources, you can ask ccm 38 | to use those by initiating a new cluster with: 39 | 40 | ccm create test --install-dir= 41 | 42 | or, from that source tree directory, simply 43 | 44 | ccm create test 45 | 46 | 2. You can ask ccm to use a released version of Cassandra. For instance to 47 | use Cassandra 2.0.5, run 48 | 49 | ccm create test -v 2.0.5 50 | 51 | ccm will download the binary (from http://archive.apache.org/dist/cassandra), 52 | and set the new cluster to use it. This means 53 | that this command can take a few minutes the first time you 54 | create a cluster for a given version. ccm saves the compiled 55 | source in `~/.ccm/repository/`, so creating a cluster for that 56 | version will be much faster the second time you run it 57 | (note however that if you create a lot of clusters with 58 | different versions, this will take up disk space). 59 | 60 | Once the cluster is created, you can populate it with 3 nodes with: 61 | 62 | ccm populate -n 3 63 | 64 | For Mac OSX, create a new interface for every node besides the first, for example if you populated your cluster with 3 nodes, create interfaces for 127.0.0.2 and 127.0.0.3 like so: 65 | 66 | sudo ifconfig lo0 alias 127.0.0.2 67 | sudo ifconfig lo0 alias 127.0.0.3 68 | 69 | Note these aliases will disappear on reboot. For permanent network aliases on Mac OSX see ![Network Aliases](./NETWORK_ALIASES.md). 70 | 71 | After that execute: 72 | 73 | ccm start 74 | 75 | That will start 3 nodes on IP 127.0.0.[1, 2, 3] on port 9160 for thrift, port 76 | 7000 for the internal cluster communication and ports 7100, 7200 and 7300 for JMX. 77 | You can check that the cluster is correctly set up with 78 | 79 | ccm node1 ring 80 | 81 | You can then bootstrap a 4th node with 82 | 83 | ccm add node4 -i 127.0.0.4 -j 7400 -b 84 | 85 | (populate is just a shortcut for adding multiple nodes initially) 86 | 87 | ccm provides a number of conveniences, like flushing all of the nodes of 88 | the cluster: 89 | 90 | ccm flush 91 | 92 | or only one node: 93 | 94 | ccm node2 flush 95 | 96 | You can also easily look at the log file of a given node with: 97 | 98 | ccm node1 showlog 99 | 100 | Finally, you can get rid of the whole cluster (which will stop the node and 101 | remove all the data) with 102 | 103 | ccm remove 104 | 105 | The list of other provided commands is available through 106 | 107 | ccm 108 | 109 | Each command is then documented through the `-h` (or `--help`) flag. For 110 | instance `ccm add -h` describes the options for `ccm add`. 111 | 112 | ### Remote Usage (SSH/Paramiko) 113 | 114 | All the usage examples above will work exactly the same for a remotely 115 | configured machine; however remote options are required in order to establish a 116 | connection to the remote machine before executing the CCM commands: 117 | 118 | | Argument | Value | Description | 119 | | :--- | :--- | :--- | 120 | | --ssh-host | string | Hostname or IP address to use for SSH connection | 121 | | --ssh-port | int | Port to use for SSH connection
Default is 22 | 122 | | --ssh-username | string | Username to use for username/password or public key authentication | 123 | | --ssh-password | string | Password to use for username/password or private key passphrase using public key authentication | 124 | | --ssh-private-key | filename | Private key to use for SSH connection | 125 | 126 | #### Special Handling 127 | 128 | Some commands require files to be located on the remote server. Those commands 129 | are pre-processed, file transfers are initiated, and updates are made to the 130 | argument value for the remote execution of the CCM command: 131 | 132 | | Parameter | Description | 133 | | :--- | :--- | 134 | | `--dse-credentials` | Copy local DSE credentials file to remote server | 135 | | `--node-ssl` | Recursively copy node SSL directory to remote server | 136 | | `--ssl` | Recursively copy SSL directory to remote server | 137 | 138 | #### Short Version 139 | 140 | ccm --ssh-host=192.168.33.11 --ssh-username=vagrant --ssh-password=vagrant create test -v 2.0.5 -n 3 -i 192.168.33.1 -s 141 | 142 | __Note__: `-i` is used to add an IP prefix during the create process to ensure 143 | that the nodes communicate using the proper IP address for their node 144 | 145 | ### Source Distribution 146 | 147 | If you'd like to use a source distribution instead of the default binary each time (for example, for Continuous Integration), you can prefix cassandra version with `source:`, for example: 148 | 149 | ``` 150 | ccm create test -v source:2.0.5 -n 3 -s 151 | ``` 152 | 153 | ### Automatic Version Fallback 154 | 155 | If 'binary:' or 'source:' are not explicitly specified in your version string, then ccm will fallback to building the requested version from git if it cannot access the apache mirrors. 156 | 157 | ### Git and GitHub 158 | 159 | To use the latest version from the [canonical Apache Git repository](https://gitbox.apache.org/repos/asf?p=cassandra.git), use the version name `git:branch-name`, e.g.: 160 | 161 | ``` 162 | ccm create trunk -v git:trunk -n 5 163 | ``` 164 | 165 | and to download a branch from a GitHub fork of Cassandra, you can prefix the repository and branch with `github:`, e.g.: 166 | 167 | ``` 168 | ccm create patched -v github:jbellis/trunk -n 1 169 | ``` 170 | 171 | ### Bash command-line completion 172 | ccm has many sub-commands for both cluster commands as well as node commands, and sometimes you don't quite remember the name of the sub-command you want to invoke. Also, command lines may be long due to long cluster or node names. 173 | 174 | Leverage bash's *programmable completion* feature to make ccm use more pleasant. Copy `misc/ccm-completion.bash` to somewhere in your home directory (or /etc if you want to make it accessible to all users of your system) and source it in your `.bash_profile`: 175 | ``` 176 | . ~/scripts/ccm-completion.bash 177 | ``` 178 | 179 | Once set up, `ccm sw` expands to `ccm switch `, for example. The `switch` sub-command has extra completion logic to help complete the cluster name. So `ccm switch cl` would expand to `ccm switch cluster-58` if cluster-58 is the only cluster whose name starts with "cl". If there is ambiguity, hitting `` a second time shows the choices that match: 180 | ``` 181 | $ ccm switch cl 182 | ... becomes ... 183 | $ ccm switch cluster- 184 | ... then hit tab twice ... 185 | cluster-56 cluster-85 cluster-96 186 | $ ccm switch cluster-8 187 | ... becomes ... 188 | $ ccm switch cluster-85 189 | ``` 190 | 191 | It dynamically determines available sub-commands based on the ccm being invoked. Thus, users running multiple ccm's (or a ccm that they are continuously updating with new commands) will automagically work. 192 | 193 | The completion script relies on ccm having two hidden subcommands: 194 | * show-cluster-cmds - emits the names of cluster sub-commands. 195 | * show-node-cmds - emits the names of node sub-commands. 196 | 197 | Thus, it will not work with sufficiently old versions of ccm. 198 | 199 | 200 | Remote debugging 201 | ----------------------- 202 | 203 | If you would like to connect to your Cassandra nodes with a remote debugger you have to pass the `-d` (or `--debug`) flag to the populate command: 204 | 205 | ccm populate -d -n 3 206 | 207 | That will populate 3 nodes on IP 127.0.0.[1, 2, 3] setting up the remote debugging on ports 2100, 2200 and 2300. 208 | The main thread will not be suspended so you don't have to connect with a remote debugger to start a node. 209 | 210 | Alternatively you can also specify a remote port with the `-r` (or `--remote-debug-port`) flag while adding a node 211 | 212 | ccm add node4 -r 5005 -i 127.0.0.4 -j 7400 -b 213 | 214 | Where things are stored 215 | ----------------------- 216 | 217 | By default, ccm stores all the node data and configuration files under `~/.ccm/cluster_name/`. 218 | This can be overridden using the `--config-dir` option with each command. 219 | 220 | 221 | 222 | Third Party Clusters 223 | ==================== 224 | 225 | `ccmlib.Cluster` and `ccmlib.Node` are designed to be extended for third party cluster types. DataStax Enterprise (DSE) and Hyper Converged Database (HCD) are example implementations. 226 | 227 | DataStax Enterprise (DSE) 228 | ------------------------- 229 | 230 | CCM 2.0 supports creating and interacting with DSE clusters. The `--dse` option must be used with the `ccm create` command. See the `ccm create -h` help for assistance. 231 | 232 | 233 | Hyper Converged Database (HCD) 234 | ------------------------------ 235 | 236 | The `--hcd` option must be used with the `ccm create` command. See the `ccm create -h` help for assistance. 237 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /ccmlib/dse/dse_cluster.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # DataStax Enterprise (DSE) clusters 19 | 20 | from __future__ import absolute_import 21 | 22 | import os 23 | import re 24 | import shutil 25 | import signal 26 | import subprocess 27 | import tarfile 28 | import tempfile 29 | from argparse import ArgumentError 30 | from distutils.version import LooseVersion 31 | from six.moves import urllib 32 | 33 | from ccmlib import common, repository 34 | from ccmlib.cluster import Cluster 35 | from ccmlib.common import rmdirs 36 | from ccmlib.common import ArgumentError 37 | from ccmlib.dse.dse_node import DseNode 38 | 39 | try: 40 | import ConfigParser 41 | except ImportError: 42 | import configparser as ConfigParser 43 | 44 | 45 | DSE_CASSANDRA_CONF_DIR = "resources/cassandra/conf" 46 | OPSCENTER_CONF_DIR = "conf" 47 | DSE_ARCHIVE = "https://downloads.datastax.com/enterprise/dse-%s-bin.tar.gz" 48 | OPSC_ARCHIVE = "https://downloads.datastax.com/enterprise/opscenter-%s.tar.gz" 49 | 50 | 51 | 52 | def isDse(install_dir, options=None): 53 | if install_dir is None: 54 | raise ArgumentError('Undefined installation directory') 55 | bin_dir = os.path.join(install_dir, common.BIN_DIR) 56 | if options and options.dse and './' != install_dir and not os.path.exists(bin_dir): 57 | raise ArgumentError('Installation directory does not contain a bin directory: %s' % install_dir) 58 | if options and options.dse: 59 | return True 60 | dse_script = os.path.join(bin_dir, 'dse') 61 | if options and not options.dse and './' != install_dir and os.path.exists(dse_script): 62 | raise ArgumentError('Installation directory is DSE but options did not specify `--dse`: %s' % install_dir) 63 | return os.path.exists(dse_script) 64 | 65 | 66 | def isOpscenter(install_dir, options=None): 67 | if install_dir is None: 68 | raise ArgumentError('Undefined installation directory') 69 | bin_dir = os.path.join(install_dir, common.BIN_DIR) 70 | if options and options.dse and './' != install_dir and not os.path.exists(bin_dir): 71 | raise ArgumentError('Installation directory does not contain a bin directory') 72 | opscenter_script = os.path.join(bin_dir, 'opscenter') 73 | return os.path.exists(opscenter_script) 74 | 75 | 76 | def isDseClusterType(install_dir, options=None): 77 | if isDse(install_dir, options) or isOpscenter(install_dir, options): 78 | return DseCluster 79 | return None 80 | 81 | 82 | class DseCluster(Cluster): 83 | 84 | @staticmethod 85 | def getConfDir(install_dir): 86 | if isDse(install_dir): 87 | return os.path.join(install_dir, DSE_CASSANDRA_CONF_DIR) 88 | elif isOpscenter(install_dir): 89 | return os.path.join(os.path.join(install_dir, OPSCENTER_CONF_DIR), common.CASSANDRA_CONF) 90 | raise RuntimeError("illegal call to DseCluster.getConfDir() when not dse or opscenter") 91 | 92 | @staticmethod 93 | def getNodeClass(): 94 | return DseNode 95 | 96 | 97 | def __init__(self, path, name, partitioner=None, install_dir=None, create_directory=True, version=None, verbose=False, derived_cassandra_version=None, options=None): 98 | self.load_credentials_from_file(options.dse_credentials_file if options else None) 99 | self.dse_username = options.dse_username if options else None 100 | self.dse_password = options.dse_password if options else None 101 | self.opscenter = options.opscenter if options else None 102 | self._cassandra_version = None 103 | self._cassandra_version = derived_cassandra_version 104 | 105 | super(DseCluster, self).__init__(path, name, partitioner, install_dir, create_directory, version, verbose, options=options) 106 | 107 | def load_from_repository(self, version, verbose): 108 | if self.opscenter is not None: 109 | odir = setup_opscenter(self.opscenter, self.dse_username, self.dse_password, verbose) 110 | target_dir = os.path.join(self.get_path(), 'opscenter') 111 | shutil.copytree(odir, target_dir) 112 | return setup_dse(version, self.dse_username, self.dse_password, verbose) 113 | 114 | def load_credentials_from_file(self, dse_credentials_file): 115 | # Use .dse.ini if it exists in the default .ccm directory. 116 | if dse_credentials_file is None: 117 | creds_file = os.path.join(common.get_default_path(), '.dse.ini') 118 | if os.path.isfile(creds_file): 119 | dse_credentials_file = creds_file 120 | 121 | if dse_credentials_file is not None: 122 | parser = ConfigParser.RawConfigParser() 123 | parser.read(dse_credentials_file) 124 | if parser.has_section('dse_credentials'): 125 | if parser.has_option('dse_credentials', 'dse_username'): 126 | self.dse_username = parser.get('dse_credentials', 'dse_username') 127 | if parser.has_option('dse_credentials', 'dse_password'): 128 | self.dse_password = parser.get('dse_credentials', 'dse_password') 129 | else: 130 | common.warning("{} does not contain a 'dse_credentials' section.".format(dse_credentials_file)) 131 | 132 | def get_seeds(self): 133 | return [s.network_interfaces['storage'][0] if isinstance(s, DseNode) else s for s in self.seeds] 134 | 135 | def hasOpscenter(self): 136 | return os.path.exists(os.path.join(self.get_path(), 'opscenter')) 137 | 138 | def create_node(self, name, auto_bootstrap, thrift_interface, storage_interface, jmx_port, remote_debug_port, initial_token, save=True, binary_interface=None, byteman_port='0', environment_variables=None,derived_cassandra_version=None): 139 | return DseNode(name, self, auto_bootstrap, thrift_interface, storage_interface, jmx_port, remote_debug_port, initial_token, save, binary_interface, byteman_port, environment_variables=environment_variables, derived_cassandra_version=derived_cassandra_version) 140 | 141 | def can_generate_tokens(self): 142 | return False 143 | 144 | def start(self, no_wait=False, verbose=False, wait_for_binary_proto=False, wait_other_notice=True, jvm_args=None, profile_options=None, quiet_start=False, allow_root=False, jvm_version=None): 145 | if jvm_args is None: 146 | jvm_args = [] 147 | marks = {} 148 | for node in self.nodelist(): 149 | marks[node] = node.mark_log() 150 | started = super(DseCluster, self).start(no_wait, verbose, wait_for_binary_proto, wait_other_notice, jvm_args, profile_options, quiet_start=quiet_start, allow_root=allow_root, timeout=180, jvm_version=jvm_version) 151 | self.start_opscenter() 152 | if self._misc_config_options.get('enable_aoss', False): 153 | self.wait_for_any_log('AlwaysOn SQL started', 600, marks=marks) 154 | return started 155 | 156 | def stop(self, wait=True, signal_event=signal.SIGTERM, **kwargs): 157 | not_running = super(DseCluster, self).stop(wait=wait, signal_event=signal.SIGTERM, **kwargs) 158 | self.stop_opscenter() 159 | return not_running 160 | 161 | def remove(self, node=None): 162 | # We _must_ gracefully stop if aoss is enabled, otherwise we will leak the spark workers 163 | super(DseCluster, self).remove(node=node, gently=self._misc_config_options.get('enable_aoss', False)) 164 | 165 | def cassandra_version(self): 166 | if self._cassandra_version is None: 167 | self._cassandra_version = get_dse_cassandra_version(self.get_install_dir()) 168 | return self._cassandra_version 169 | 170 | def enable_aoss(self): 171 | if self.version() < '6.0': 172 | common.error("Cannot enable AOSS in DSE clusters before 6.0") 173 | exit(1) 174 | self._misc_config_options['enable_aoss'] = True 175 | for node in self.nodelist(): 176 | port_offset = int(node.name[4:]) 177 | node.enable_aoss(thrift_port=10000 + port_offset, web_ui_port=9077 + port_offset) 178 | self._update_config() 179 | 180 | def set_dse_configuration_options(self, values=None): 181 | if values is not None: 182 | self._dse_config_options = common.merge_configuration(self._dse_config_options, values) 183 | self._update_config() 184 | for node in list(self.nodes.values()): 185 | node.import_dse_config_files() 186 | return self 187 | 188 | def start_opscenter(self): 189 | if self.hasOpscenter(): 190 | self.write_opscenter_cluster_config() 191 | args = [os.path.join(self.get_path(), 'opscenter', 'bin', common.platform_binary('opscenter'))] 192 | subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 193 | 194 | def stop_opscenter(self): 195 | pidfile = os.path.join(self.get_path(), 'opscenter', 'twistd.pid') 196 | if os.path.exists(pidfile): 197 | with open(pidfile, 'r') as f: 198 | pid = int(f.readline().strip()) 199 | f.close() 200 | if pid is not None: 201 | try: 202 | os.kill(pid, signal.SIGKILL) 203 | except OSError: 204 | pass 205 | os.remove(pidfile) 206 | 207 | def write_opscenter_cluster_config(self): 208 | cluster_conf = os.path.join(self.get_path(), 'opscenter', 'conf', 'clusters') 209 | if not os.path.exists(cluster_conf): 210 | os.makedirs(cluster_conf) 211 | if len(self.nodes) > 0: 212 | node = list(self.nodes.values())[0] 213 | (node_ip, node_port) = node.network_interfaces['thrift'] 214 | node_jmx = node.jmx_port 215 | with open(os.path.join(cluster_conf, self.name + '.conf'), 'w+') as f: 216 | f.write('[jmx]\n') 217 | f.write('port = %s\n' % node_jmx) 218 | f.write('[cassandra]\n') 219 | f.write('seed_hosts = %s\n' % node_ip) 220 | f.write('api_port = %s\n' % node_port) 221 | f.close() 222 | 223 | 224 | def setup_dse(version, username, password, verbose=False): 225 | (cdir, version, fallback) = repository.__setup(version, verbose) 226 | if cdir: 227 | return (cdir, version) 228 | cdir = repository.version_directory(version) 229 | if cdir is None: 230 | download_dse_version(version, username, password, verbose=verbose) 231 | cdir = repository.version_directory(version) 232 | return (cdir, version) 233 | 234 | 235 | def setup_opscenter(opscenter, username, password, verbose=False): 236 | ops_version = 'opsc' + opscenter 237 | odir = repository.version_directory(ops_version) 238 | if odir is None: 239 | download_opscenter_version(opscenter, username, password, ops_version, verbose=verbose) 240 | odir = repository.version_directory(ops_version) 241 | return odir 242 | 243 | 244 | def get_dse_cassandra_version(install_dir): 245 | # for this to work, the current JAVA_HOME must already be appropriate 246 | dse_cmd = os.path.join(install_dir, 'bin', 'dse') 247 | (output, stderr) = subprocess.Popen([dse_cmd, "cassandra", '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() 248 | # just take the last line to avoid any possible log lines 249 | output = output.decode('utf-8').rstrip().split('\n')[-1] 250 | match = re.search('^([0-9.]+)(?:-.*)?', str(output)) 251 | if match: 252 | return LooseVersion(match.group(1)) 253 | raise ArgumentError("Unable to determine Cassandra version using `bin/dse cassandra -v` from output: %s.\n\tstdout: '%s'\n\tstderr: '%s'" 254 | % (install_dir, output, stderr)) 255 | 256 | 257 | def download_dse_version(version, username, password, verbose=False): 258 | url = DSE_ARCHIVE 259 | if repository.CCM_CONFIG.has_option('repositories', 'dse'): 260 | url = repository.CCM_CONFIG.get('repositories', 'dse') 261 | 262 | url = url % version 263 | _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") 264 | try: 265 | if username is None: 266 | common.warning("No dse username detected, specify one using --dse-username or passing in a credentials file using --dse-credentials.") 267 | if password is None: 268 | common.warning("No dse password detected, specify one using --dse-password or passing in a credentials file using --dse-credentials.") 269 | repository.__download(url, target, username=username, password=password, show_progress=verbose) 270 | common.debug("Extracting {} as version {} ...".format(target, version)) 271 | tar = tarfile.open(target) 272 | dir = tar.next().name.split("/")[0] # pylint: disable=all 273 | tar.extractall(path=repository.__get_dir()) 274 | tar.close() 275 | target_dir = os.path.join(repository.__get_dir(), version) 276 | if os.path.exists(target_dir): 277 | rmdirs(target_dir) 278 | shutil.move(os.path.join(repository.__get_dir(), dir), target_dir) 279 | except urllib.error.URLError as e: 280 | msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url 281 | msg = msg + " (underlying error is: %s)" % str(e) 282 | raise ArgumentError(msg) 283 | except tarfile.ReadError as e: 284 | raise ArgumentError("Unable to uncompress downloaded file: %s" % str(e)) 285 | 286 | 287 | def download_opscenter_version(version, username, password, target_version, verbose=False): 288 | url = OPSC_ARCHIVE 289 | if repository.CCM_CONFIG.has_option('repositories', 'opscenter'): 290 | url = repository.CCM_CONFIG.get('repositories', 'opscenter') 291 | 292 | url = url % version 293 | _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") 294 | try: 295 | if username is None: 296 | common.warning("No dse username detected, specify one using --dse-username or passing in a credentials file using --dse-credentials.") 297 | if password is None: 298 | common.warning("No dse password detected, specify one using --dse-password or passing in a credentials file using --dse-credentials.") 299 | repository.__download(url, target, username=username, password=password, show_progress=verbose) 300 | common.info("Extracting {} as version {} ...".format(target, target_version)) 301 | tar = tarfile.open(target) 302 | dir = tar.next().name.split("/")[0] # pylint: disable=all 303 | tar.extractall(path=repository.__get_dir()) 304 | tar.close() 305 | target_dir = os.path.join(repository.__get_dir(), target_version) 306 | if os.path.exists(target_dir): 307 | rmdirs(target_dir) 308 | shutil.move(os.path.join(repository.__get_dir(), dir), target_dir) 309 | except urllib.error.URLError as e: 310 | msg = "Invalid version {}".format(version) if url is None else "Invalid url {}".format(url) 311 | msg = msg + " (underlying error is: {})".format(str(e)) 312 | raise ArgumentError(msg) 313 | except tarfile.ReadError as e: 314 | raise ArgumentError("Unable to uncompress downloaded file: {}".format(str(e))) 315 | -------------------------------------------------------------------------------- /ccmlib/remote.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # 19 | # Remote execution helper functionality for executing CCM commands on a remote 20 | # machine with CCM installed 21 | # 22 | 23 | from __future__ import absolute_import 24 | 25 | import argparse 26 | import logging 27 | import os 28 | import re 29 | import select 30 | import stat 31 | import sys 32 | import tempfile 33 | 34 | # Paramiko is an optional library as SSH is optional for CCM 35 | PARAMIKO_IS_AVAILABLE = False 36 | try: 37 | import paramiko 38 | PARAMIKO_IS_AVAILABLE = True 39 | except ImportError: 40 | pass 41 | 42 | 43 | def get_remote_usage(): 44 | """ 45 | Get the usage for the remote exectuion options 46 | 47 | :return Usage for the remote execution options 48 | """ 49 | return RemoteOptionsParser().usage() 50 | 51 | 52 | def get_remote_options(): 53 | """ 54 | Parse the command line arguments and split out the CCM arguments and the remote options 55 | 56 | :return: A tuple defining the arguments parsed and actions to take 57 | * remote_options - Remote options only 58 | * ccm_args - CCM arguments only 59 | :raises Exception if private key is not a file 60 | """ 61 | return RemoteOptionsParser().parse_known_options() 62 | 63 | 64 | def execute_ccm_remotely(remote_options, ccm_args): 65 | """ 66 | Execute CCM operation(s) remotely 67 | 68 | :return A tuple defining the execution of the command 69 | * output - The output of the execution if the output was not displayed 70 | * exit_status - The exit status of remotely executed script 71 | :raises Exception if invalid options are passed for `--dse-credentials`, `--ssl`, or 72 | `--node-ssl` when initiating a remote execution; also if 73 | error occured during ssh connection 74 | """ 75 | if not PARAMIKO_IS_AVAILABLE: 76 | logging.warn("Paramiko is not Availble: Skipping remote execution of CCM command") 77 | return None, None 78 | 79 | # Create the SSH client 80 | ssh_client = SSHClient(remote_options.ssh_host, remote_options.ssh_port, 81 | remote_options.ssh_username, remote_options.ssh_password, 82 | remote_options.ssh_private_key) 83 | 84 | # Handle CCM arguments that require SFTP 85 | for index, argument in enumerate(ccm_args): 86 | # Determine if DSE credentials argument is being used 87 | if "--dse-credentials" in argument: 88 | # Get the filename being used for the DSE credentials 89 | tokens = argument.split("=") 90 | credentials_path = os.path.join(os.path.expanduser("~"), ".ccm", ".dse.ini") 91 | if len(tokens) == 2: 92 | credentials_path = tokens[1] 93 | 94 | # Ensure the credential file exists locally and copy to remote host 95 | if not os.path.isfile(credentials_path): 96 | raise Exception("DSE Credentials File Does not Exist: %s" 97 | % credentials_path) 98 | ssh_client.put(credentials_path, ssh_client.ccm_config_dir) 99 | 100 | # Update the DSE credentials argument 101 | ccm_args[index] = "--dse-credentials" 102 | 103 | # Determine if SSL or node SSL path argument is being used 104 | if "--ssl" in argument or "--node-ssl" in argument: 105 | # Get the directory being used for the path 106 | tokens = argument.split("=") 107 | if len(tokens) != 2: 108 | raise Exception("Path is not Specified: %s" % argument) 109 | ssl_path = tokens[1] 110 | 111 | # Ensure the path exists locally and copy to remote host 112 | if not os.path.isdir(ssl_path): 113 | raise Exception("Path Does not Exist: %s" % ssl_path) 114 | remote_ssl_path = ssh_client.temp + os.path.basename(ssl_path) 115 | ssh_client.put(ssl_path, remote_ssl_path) 116 | 117 | # Update the argument 118 | ccm_args[index] = tokens[0] + "=" + remote_ssl_path 119 | 120 | # Execute the CCM request, return output and exit status 121 | return ssh_client.execute_ccm_command(ccm_args) 122 | 123 | 124 | class SSHClient: 125 | """ 126 | SSH client class used to handle SSH operations to the remote host 127 | """ 128 | 129 | def __init__(self, host, port, username, password, private_key=None): 130 | """ 131 | Create the SSH client 132 | 133 | :param host: Hostname or IP address to connect to 134 | :param port: Port number to use for SSH 135 | :param username: Username credentials for SSH access 136 | :param password: Password credentials for SSH access (or private key passphrase) 137 | :param private_key: Private key to bypass clear text password (default: None - Username and 138 | password credentials) 139 | """ 140 | # Reduce the noise from the logger for paramiko 141 | logging.getLogger("paramiko").setLevel(logging.WARNING) 142 | 143 | # Establish the SSH connection 144 | self.ssh = self.__connect(host, port, username, password, private_key) 145 | 146 | # Gather information about the remote OS 147 | information = self.__server_information() 148 | self.separator = information[1] 149 | self.home = information[0] + self.separator 150 | self.temp = information[2] + self.separator 151 | self.platform = information[3] 152 | self.profile = information[4] 153 | self.distribution = information[5] 154 | 155 | # Create the CCM configuration directory variable 156 | self.ccm_config_dir = self.home + self.separator + ".ccm" + self.separator 157 | 158 | @staticmethod 159 | def __connect(host, port, username, password, private_key): 160 | """ 161 | Establish remote connection 162 | 163 | :param host: Hostname or IP address to connect to 164 | :param port: Port number to use for SSH 165 | :param username: Username credentials for SSH access 166 | :param password: Password credentials for SSH access (or private key passphrase) 167 | :param private_key: Private key to bypass clear text password 168 | :return: Paramiko SSH client instance if connection was established 169 | :raises Exception if connection was unsuccessful 170 | """ 171 | # Initialize the SSH connection 172 | ssh = paramiko.SSHClient() 173 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 174 | if private_key is not None and password is not None: 175 | private_key = paramiko.RSAKey.from_private_key_file(private_key, password) 176 | elif private_key is not None: 177 | private_key = paramiko.RSAKey.from_private_key_file(private_key, password) 178 | 179 | # Establish the SSH connection 180 | try: 181 | ssh.connect(host, port, username, password, private_key) 182 | except Exception as e: 183 | raise e 184 | 185 | # Return the established SSH connection 186 | return ssh 187 | 188 | def execute(self, command, is_displayed=True, profile=None): 189 | """ 190 | Execute a command on the remote server 191 | 192 | :param command: Command to execute remotely 193 | :param is_displayed: True if information should be display; false to return output 194 | (default: true) 195 | :param profile: Profile to source (unix like system only should set this) 196 | (default: None) 197 | :return: A tuple defining the execution of the command 198 | * output - The output of the execution if the output was not displayed 199 | * exit_status - The exit status of remotely executed script 200 | """ 201 | # Modify the command for remote execution 202 | command = " ".join("'{0}'".format(argument) for argument in command) 203 | 204 | # Execute the command and initialize for reading (close stdin/writes) 205 | if not profile is None and not profile is "None": 206 | command = "source " + profile + ";" + command 207 | stdin, stdout, stderr = self.ssh.exec_command(command) 208 | stdin.channel.shutdown_write() 209 | stdin.close() 210 | 211 | 212 | # Print or gather output as is occurs 213 | output = None 214 | if not is_displayed: 215 | output = [] 216 | output.append(stdout.channel.recv(len(stdout.channel.in_buffer)).decode("utf-8")) 217 | output.append(stderr.channel.recv(len(stderr.channel.in_buffer)).decode("utf-8")) 218 | channel = stdout.channel 219 | while not channel.closed or channel.recv_ready() or channel.recv_stderr_ready(): 220 | # Ensure the channel was not closed prematurely and all data has been ready 221 | is_data_present = False 222 | handles = select.select([channel], [], []) 223 | for read in handles[0]: 224 | # Read stdout and/or stderr if data is present 225 | buffer = None 226 | if read.recv_ready(): 227 | buffer = channel.recv(len(read.in_buffer)).decode("utf-8") 228 | if is_displayed: 229 | sys.stdout.write(buffer) 230 | if read.recv_stderr_ready(): 231 | buffer = stderr.channel.recv_stderr(len(read.in_stderr_buffer)).decode("utf-8") 232 | if is_displayed: 233 | sys.stderr.write(buffer) 234 | 235 | # Determine if the output should be updated and displayed 236 | if buffer is not None: 237 | is_data_present = True 238 | if not is_displayed: 239 | output.append(buffer) 240 | 241 | # Ensure all the data has been read and exit loop if completed 242 | if (not is_data_present and channel.exit_status_ready() 243 | and not stderr.channel.recv_stderr_ready() 244 | and not channel.recv_ready()): 245 | # Stop reading and close the channel to stop processing 246 | channel.shutdown_read() 247 | channel.close() 248 | break 249 | 250 | # Close file handles for stdout and stderr 251 | stdout.close() 252 | stderr.close() 253 | 254 | # Process the output (if available) 255 | if output is not None: 256 | output = "".join(output) 257 | 258 | # Return the output from the executed command 259 | return output, channel.recv_exit_status() 260 | 261 | def execute_ccm_command(self, ccm_args, is_displayed=True): 262 | """ 263 | Execute a CCM command on the remote server 264 | 265 | :param ccm_args: CCM arguments to execute remotely 266 | :param is_displayed: True if information should be display; false to return output 267 | (default: true) 268 | :return: A tuple defining the execution of the command 269 | * output - The output of the execution if the output was not displayed 270 | * exit_status - The exit status of remotely executed script 271 | """ 272 | return self.execute(["ccm"] + ccm_args, profile=self.profile) 273 | 274 | def execute_python_script(self, script): 275 | """ 276 | Execute a python script of the remote server 277 | 278 | :param script: Inline script to convert to a file and execute remotely 279 | :return: The output of the script execution 280 | """ 281 | # Create the local file to copy to remote 282 | file_handle, filename = tempfile.mkstemp() 283 | temp_file = os.fdopen(file_handle, "wt") 284 | temp_file.write(script) 285 | temp_file.close() 286 | 287 | # Put the file into the remote user directory 288 | self.put(filename, "python_execute.py") 289 | command = ["python", "python_execute.py"] 290 | 291 | # Execute the python script on the remote system, clean up, and return the output 292 | output = self.execute(command, False) 293 | self.remove("python_execute.py") 294 | os.unlink(filename) 295 | return output 296 | 297 | def put(self, local_path, remote_path=None): 298 | """ 299 | Copy a file (or directory recursively) to a location on the remote server 300 | 301 | :param local_path: Local path to copy to; can be file or directory 302 | :param remote_path: Remote path to copy to (default: None - Copies file or directory to 303 | home directory directory on the remote server) 304 | """ 305 | # Determine if local_path should be put into remote user directory 306 | if remote_path is None: 307 | remote_path = os.path.basename(local_path) 308 | 309 | ftp = self.ssh.open_sftp() 310 | if os.path.isdir(local_path): 311 | self.__put_dir(ftp, local_path, remote_path) 312 | else: 313 | ftp.put(local_path, remote_path) 314 | ftp.close() 315 | 316 | def __put_dir(self, ftp, local_path, remote_path=None): 317 | """ 318 | Helper function to perform copy operation to remote server 319 | 320 | :param ftp: SFTP handle to perform copy operation(s) 321 | :param local_path: Local path to copy to; can be file or directory 322 | :param remote_path: Remote path to copy to (default: None - Copies file or directory to 323 | home directory directory on the remote server) 324 | """ 325 | # Determine if local_path should be put into remote user directory 326 | if remote_path is None: 327 | remote_path = os.path.basename(local_path) 328 | remote_path += self.separator 329 | 330 | # Iterate over the local path and perform copy operations to remote server 331 | for current_path, directories, files in os.walk(local_path): 332 | # Create the remote directory (if needed) 333 | try: 334 | ftp.listdir(remote_path) 335 | except IOError: 336 | ftp.mkdir(remote_path) 337 | 338 | # Copy the files in the current directory to the remote path 339 | for filename in files: 340 | ftp.put(os.path.join(current_path, filename), remote_path + filename) 341 | # Copy the directory in the current directory to the remote path 342 | for directory in directories: 343 | self.__put_dir(ftp, os.path.join(current_path, directory), remote_path + directory) 344 | 345 | def remove(self, remote_path): 346 | """ 347 | Delete a file or directory recursively on the remote server 348 | 349 | :param remote_path: Remote path to remove 350 | """ 351 | # Based on the remote file stats; remove a file or directory recursively 352 | ftp = self.ssh.open_sftp() 353 | if stat.S_ISDIR(ftp.stat(remote_path).st_mode): 354 | self.__remove_dir(ftp, remote_path) 355 | else: 356 | ftp.remove(remote_path) 357 | ftp.close() 358 | 359 | def __remove_dir(self, ftp, remote_path): 360 | """ 361 | Helper function to perform delete operation on the remote server 362 | 363 | :param ftp: SFTP handle to perform delete operation(s) 364 | :param remote_path: Remote path to remove 365 | """ 366 | # Iterate over the remote path and perform remove operations 367 | files = ftp.listdir(remote_path) 368 | for filename in files: 369 | # Attempt to remove the file (if exception then path is directory) 370 | path = remote_path + self.separator + filename 371 | try: 372 | ftp.remove(path) 373 | except IOError: 374 | self.__remove_dir(ftp, path) 375 | 376 | # Remove the original directory requested 377 | ftp.rmdir(remote_path) 378 | 379 | def __server_information(self): 380 | """ 381 | Get information about the remote server: 382 | * User's home directory 383 | * OS separator 384 | * OS temporary directory 385 | * Platform information 386 | * Profile to source (Available only on unix-like platforms (including Mac OS)) 387 | * Platform distribution ((Available only on unix-like platforms) 388 | :return: Remote executed script with the above information (line by line in order) 389 | """ 390 | return self.execute_python_script("""import os 391 | import platform 392 | import sys 393 | import tempfile 394 | 395 | # Standard system information 396 | print(os.path.expanduser("~")) 397 | print(os.sep) 398 | print(tempfile.gettempdir()) 399 | print(platform.system()) 400 | 401 | # Determine the profile for unix like systems 402 | if sys.platform == "darwin" or sys.platform == "linux" or sys.platform == "linux2": 403 | if os.path.isfile(".profile"): 404 | print(os.path.expanduser("~") + os.sep + ".profile") 405 | elif os.path.isfile(".bash_profile"): 406 | print(os.path.expanduser("~") + os.sep + ".bash_profile") 407 | else: 408 | print("None") 409 | else: 410 | print("None") 411 | 412 | # Get the disto information for unix like system (excluding Mac OS) 413 | if sys.platform == "linux" or sys.platform == "linux2": 414 | print(platform.linux_distribution()) 415 | else: 416 | print("None") 417 | """)[0].splitlines() 418 | 419 | 420 | class RemoteOptionsParser(): 421 | """ 422 | Parser class to facilitate remote execution of CCM operations via command 423 | line arguments 424 | """ 425 | 426 | def __init__(self): 427 | """ 428 | Create the parser for the remote CCM operations allowed 429 | """ 430 | self.parser = argparse.ArgumentParser(description="Remote", 431 | add_help=False) 432 | 433 | # Add the SSH arguments for the remote parser 434 | self.parser.add_argument( 435 | "--ssh-host", 436 | default=None, 437 | type=str, 438 | help="Hostname or IP address to use for SSH connection" 439 | ) 440 | self.parser.add_argument( 441 | "--ssh-port", 442 | default=22, 443 | type=self.port, 444 | help="Port to use for SSH connection" 445 | ) 446 | self.parser.add_argument( 447 | "--ssh-username", 448 | default=None, 449 | type=str, 450 | help="Username to use for username/password or public key authentication" 451 | ) 452 | self.parser.add_argument( 453 | "--ssh-password", 454 | default=None, 455 | type=str, 456 | help="Password to use for username/password or private key passphrase using public " 457 | "key authentication" 458 | ) 459 | self.parser.add_argument( 460 | "--ssh-private-key", 461 | default=None, 462 | type=self.ssh_key, 463 | help="Private key to use for SSH connection" 464 | ) 465 | 466 | def parse_known_options(self): 467 | """ 468 | Parse the command line arguments and split out the remote options and CCM arguments 469 | 470 | :return: A tuple defining the arguments parsed and actions to take 471 | * remote_options - Remote options only 472 | * ccm_args - CCM arguments only 473 | """ 474 | # Parse the known arguments and return the remote options and CCM arguments 475 | remote_options, ccm_args = self.parser.parse_known_args() 476 | return remote_options, ccm_args 477 | 478 | @staticmethod 479 | def ssh_key(key): 480 | """ 481 | SSH key parser validator (ensure file exists) 482 | 483 | :param key: Filename/Key to validate (ensure exists) 484 | :return: The filename/key passed in (if valid) 485 | :raises Exception if filename/key is not a valid file 486 | """ 487 | value = str(key) 488 | # Ensure the file exists locally 489 | if not os.path.isfile(value): 490 | raise Exception("File Does not Exist: %s" % key) 491 | return value 492 | 493 | @staticmethod 494 | def port(port): 495 | """ 496 | Port validator 497 | 498 | :param port: Port to validate (1 - 65535) 499 | :return: Port passed in (if valid) 500 | :raises ArgumentTypeError if port is not valid 501 | """ 502 | value = int(port) 503 | if value <= 0 or value > 65535: 504 | raise argparse.ArgumentTypeError("%s must be between [1 - 65535]" % port) 505 | return value 506 | 507 | def usage(self): 508 | """ 509 | Get the usage for the remote exectuion options 510 | 511 | :return Usage for the remote execution options 512 | """ 513 | remote_arguments_usage = self.parser.format_help() 514 | chunks = re.split("^(optional arguments:|options:)", remote_arguments_usage) 515 | if len(chunks) > 2: 516 | # Try to extract argument description only, otherwise output the whole usage string 517 | remote_arguments_usage = chunks[2] 518 | # Remove any blank lines and return 519 | return "Remote Options:" + os.linesep + \ 520 | os.linesep.join([s for s in remote_arguments_usage.splitlines() if s]) 521 | -------------------------------------------------------------------------------- /ccmlib/repository.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # downloaded sources handling 19 | from __future__ import absolute_import, division, with_statement 20 | 21 | import json 22 | import logging 23 | from logging import handlers 24 | import os 25 | import re 26 | import shutil 27 | import stat 28 | import subprocess 29 | import sys 30 | import tarfile 31 | import tempfile 32 | import time 33 | from distutils.version import LooseVersion # pylint: disable=import-error, no-name-in-module 34 | 35 | from six import next, print_ 36 | 37 | try: 38 | import ConfigParser 39 | except ImportError: 40 | import configparser as ConfigParser 41 | 42 | from ccmlib import common 43 | from ccmlib.common import (ArgumentError, CCMError, 44 | update_java_version, get_default_path, get_jdk_version_int, 45 | platform_binary, rmdirs, validate_install_dir) 46 | from six.moves import urllib 47 | 48 | 49 | ARCHIVE = "http://archive.apache.org/dist/cassandra" 50 | GIT_REPO = "https://github.com/apache/cassandra.git" 51 | GITHUB_REPO = "https://github.com/apache/cassandra" 52 | GITHUB_TAGS = "https://api.github.com/repos/apache/cassandra/git/refs/tags" 53 | CCM_CONFIG = ConfigParser.RawConfigParser() 54 | CCM_CONFIG.read(os.path.join(os.path.expanduser("~"), ".ccm", "config")) 55 | 56 | 57 | def setup(version, verbose=False): 58 | (cdir, version, fallback) = __setup(version, verbose) 59 | if cdir: 60 | return (cdir, version) 61 | 62 | if version in ('stable', 'oldstable', 'testing'): 63 | version = get_tagged_version_numbers(version)[0] 64 | 65 | cdir = version_directory(version) 66 | if cdir is None: 67 | try: 68 | download_version(version, verbose=verbose, binary=True) 69 | cdir = version_directory(version) 70 | except Exception as e: 71 | # If we failed to download from ARCHIVE, 72 | # then we build from source from the git repo, 73 | # as it is more reliable. 74 | # We don't do this if binary: or source: were 75 | # explicitly specified. 76 | if fallback: 77 | common.warning("Downloading {} failed, trying to build from git instead.\n" 78 | "The error was: {}".format(version, e)) 79 | version = 'git:cassandra-{}'.format(version) 80 | clone_development(GIT_REPO, version, verbose=verbose) 81 | return (version_directory(version), None) 82 | else: 83 | raise e 84 | return (cdir, version) 85 | 86 | 87 | def __setup(version, verbose=False): 88 | fallback = True 89 | 90 | if version.startswith('git:'): 91 | clone_development(GIT_REPO, version, verbose=verbose) 92 | return (version_directory(version), None, fallback) 93 | elif version.startswith('local:'): 94 | # local: slugs take the form of: "local:/some/path/:somebranch" 95 | try: 96 | _, path, branch = version.split(':') 97 | except ValueError: 98 | raise CCMError("local version ({}) appears to be invalid. Please format as local:/some/path/:somebranch".format(version)) 99 | 100 | clone_development(path, version, verbose=verbose) 101 | version_dir = version_directory(version) 102 | 103 | if version_dir is None: 104 | raise CCMError("Path provided in local slug appears invalid ({})".format(path)) 105 | return (version_dir, None, fallback) 106 | 107 | elif version.startswith('binary:'): 108 | version = version.replace('binary:', '') 109 | fallback = False 110 | 111 | elif version.startswith('github:'): 112 | user_name, _ = github_username_and_branch_name(version) 113 | # make sure to use http for cloning read-only repos such as 'github:apache/cassandra-2.1' 114 | if user_name == "apache": 115 | clone_development(GITHUB_REPO, version, verbose=verbose) 116 | else: 117 | clone_development(github_repo_for_user(user_name), version, verbose=verbose) 118 | return (directory_name(version), None, fallback) 119 | 120 | elif version.startswith('source:'): 121 | version = version.replace('source:', '') 122 | 123 | elif version.startswith('clone:'): 124 | # locally present C* source tree 125 | version = version.replace('clone:', '') 126 | return (version, None, fallback) 127 | 128 | elif version.startswith('alias:'): 129 | alias = version.split(":")[1].split("/")[0] 130 | try: 131 | git_repo = CCM_CONFIG.get("aliases", alias) 132 | clone_development(git_repo, version, verbose=verbose, alias=True) 133 | return (directory_name(version), None, fallback) 134 | except ConfigParser.NoOptionError as e: 135 | common.warning("Unable to find alias {} in configuration file.".format(alias)) 136 | raise e 137 | return (None, version, fallback) 138 | 139 | 140 | def validate(path): 141 | if path.startswith(__get_dir()): 142 | _, version = os.path.split(os.path.normpath(path)) 143 | setup(version) 144 | 145 | 146 | def clone_development(git_repo, version, verbose=False, alias=False): 147 | print_(git_repo, version) 148 | target_dir = directory_name(version) 149 | assert target_dir 150 | if 'github' in version: 151 | git_repo_name, git_branch = github_username_and_branch_name(version) 152 | elif 'local:' in version: 153 | git_repo_name = 'local_{}'.format(git_repo) # add git repo location to distinguish cache location for differing repos 154 | git_branch = version.split(':')[-1] # last token on 'local:...' slugs should always be branch name 155 | elif alias: 156 | git_repo_name = 'alias_{}'.format(version.split('/')[0].split(':')[-1]) 157 | git_branch = version.split('/')[-1] 158 | else: 159 | git_repo_name = 'apache' 160 | git_branch = version.split(':', 1)[1] 161 | local_git_cache = os.path.join(__get_dir(), '_git_cache_' + git_repo_name) 162 | 163 | logfile = lastlogfilename() 164 | logger = get_logger(logfile) 165 | 166 | try: 167 | # Checkout/fetch a local repository cache to reduce the number of 168 | # remote fetches we need to perform: 169 | if not os.path.exists(local_git_cache): 170 | common.info("Cloning Cassandra...") 171 | process = subprocess.Popen( 172 | ['git', 'clone', '--bare', 173 | git_repo, local_git_cache], 174 | cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) 175 | out, _, _ = log_info(process, logger) 176 | assert out == 0, "Could not do a git clone" 177 | else: 178 | common.info("Fetching Cassandra updates...") 179 | process = subprocess.Popen( 180 | ['git', 'fetch', '-fup', 'origin', 181 | '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*'], 182 | cwd=local_git_cache, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 183 | out, _, _ = log_info(process, logger) 184 | assert out == 0, "Could not update git" 185 | 186 | # Checkout the version we want from the local cache: 187 | if not os.path.exists(target_dir): 188 | # development branch doesn't exist. Check it out. 189 | common.info("Cloning Cassandra (from local cache)") 190 | 191 | # git on cygwin appears to be adding `cwd` to the commands which is breaking clone 192 | if sys.platform == "cygwin": 193 | local_split = local_git_cache.split(os.sep) 194 | target_split = target_dir.split(os.sep) 195 | process = subprocess.Popen( 196 | ['git', 'clone', local_split[-1], target_split[-1]], 197 | cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) 198 | out, _, _ = log_info(process, logger) 199 | assert out == 0, "Could not do a git clone" 200 | else: 201 | process = subprocess.Popen( 202 | ['git', 'clone', local_git_cache, target_dir], 203 | cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) 204 | out, _, _ = log_info(process, logger) 205 | assert out == 0, "Could not do a git clone" 206 | 207 | # determine if the request is for a branch 208 | is_branch = False 209 | try: 210 | branch_listing = subprocess.check_output(['git', 'branch', '--all'], cwd=target_dir).decode('utf-8') 211 | branches = [b.strip() for b in branch_listing.replace('remotes/origin/', '').split()] 212 | is_branch = git_branch in branches 213 | except subprocess.CalledProcessError as cpe: 214 | common.error("Error Running Branch Filter: {}\nAssumming request is not for a branch".format(cpe.output)) 215 | 216 | # now check out the right version 217 | branch_or_sha_tag = 'branch' if is_branch else 'SHA/tag' 218 | common.info("Checking out requested {} ({})".format(branch_or_sha_tag, git_branch)) 219 | if is_branch: 220 | # we use checkout -B with --track so we can specify that we want to track a specific branch 221 | # otherwise, you get errors on branch names that are also valid SHAs or SHA shortcuts, like 10360 222 | # we use -B instead of -b so we reset branches that already exist and create a new one otherwise 223 | process = subprocess.Popen(['git', 'checkout', '-B', git_branch, 224 | '--track', 'origin/{git_branch}'.format(git_branch=git_branch)], 225 | cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 226 | out, _, _ = log_info(process, logger) 227 | else: 228 | process = subprocess.Popen( 229 | ['git', 'checkout', git_branch], 230 | cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 231 | out, _, _ = log_info(process, logger) 232 | if int(out) != 0: 233 | raise CCMError('Could not check out git branch {branch}. ' 234 | 'Is this a valid branch name? (see {lastlog} or run ' 235 | '"ccm showlastlog" for details)'.format( 236 | branch=git_branch, lastlog=logfile 237 | )) 238 | # now compile 239 | compile_version(git_branch, target_dir, verbose) 240 | else: # branch is already checked out. See if it is behind and recompile if needed. 241 | process = subprocess.Popen( 242 | ['git', 'fetch', 'origin'], 243 | cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 244 | out, _, _ = log_info(process, logger) 245 | assert out == 0, "Could not do a git fetch" 246 | process = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 247 | _, status, _ = log_info(process, logger) 248 | if str(status).find('[behind') > -1: # If `status` looks like '## cassandra-2.2...origin/cassandra-2.2 [behind 9]\n' 249 | common.info("Branch is behind, recompiling") 250 | process = subprocess.Popen(['git', 'pull'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 251 | out, _, _ = log_info(process, logger) 252 | assert out == 0, "Could not do a git pull" 253 | process = subprocess.Popen([platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 254 | out, _, _ = log_info(process, logger) 255 | assert out == 0, "Could not run 'ant realclean'" 256 | 257 | # now compile 258 | compile_version(git_branch, target_dir, verbose) 259 | elif re.search('\[.*?(ahead|behind).*?\]', status.decode("utf-8")) is not None: # status looks like '## trunk...origin/trunk [ahead 1, behind 29]\n' 260 | # If we have diverged in a way that fast-forward merging cannot solve, raise an exception so the cache is wiped 261 | common.error("Could not ascertain branch status, please resolve manually.") 262 | raise Exception 263 | else: # status looks like '## cassandra-2.2...origin/cassandra-2.2\n' 264 | common.debug("Branch up to date, not pulling.") 265 | except Exception as e: 266 | # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. 267 | try: 268 | rmdirs(target_dir) 269 | common.error("Deleted {} due to error".format(target_dir)) 270 | except: 271 | print_('Building C* version {version} failed. Attempted to delete {target_dir} ' 272 | 'but failed. This will need to be manually deleted'.format( 273 | version=version, 274 | target_dir=target_dir 275 | )) 276 | finally: 277 | raise e 278 | 279 | 280 | def download_version(version, url=None, verbose=False, binary=False): 281 | """Download, extract, and build Cassandra tarball. 282 | 283 | if binary == True, download precompiled tarball, otherwise build from source tarball. 284 | """ 285 | 286 | archive_url = ARCHIVE 287 | if CCM_CONFIG.has_option('repositories', 'cassandra'): 288 | archive_url = CCM_CONFIG.get('repositories', 'cassandra') 289 | 290 | if binary: 291 | archive_url = "%s/%s/apache-cassandra-%s-bin.tar.gz" % (archive_url, version, version) if url is None else url 292 | else: 293 | archive_url = "%s/%s/apache-cassandra-%s-src.tar.gz" % (archive_url, version, version) if url is None else url 294 | _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") 295 | try: 296 | __download(archive_url, target, show_progress=verbose) 297 | common.info("Extracting {} as version {} ...".format(target, version)) 298 | tar = tarfile.open(target) 299 | dir = tar.next().name.split("/")[0] # pylint: disable=all 300 | tar.extractall(path=__get_dir()) 301 | tar.close() 302 | target_dir = os.path.join(__get_dir(), version) 303 | if os.path.exists(target_dir): 304 | rmdirs(target_dir) 305 | shutil.move(os.path.join(__get_dir(), dir), target_dir) 306 | 307 | if binary: 308 | # Binary installs don't have a build.xml that is needed 309 | # for pulling the version from. Write the version number 310 | # into a file to read later in common.get_version_from_build() 311 | with open(os.path.join(target_dir, '0.version.txt'), 'w') as f: 312 | f.write(version) 313 | else: 314 | compile_version(version, target_dir, verbose=verbose) 315 | 316 | except urllib.error.URLError as e: 317 | msg = "Invalid version {}".format(version) if url is None else "Invalid url {}".format(url) 318 | msg = msg + " (underlying error is: {})".format(str(e)) 319 | raise ArgumentError(msg) 320 | except tarfile.ReadError as e: 321 | raise ArgumentError("Unable to uncompress downloaded file: {}".format(str(e))) 322 | except CCMError as e: 323 | # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. 324 | try: 325 | rmdirs(target_dir) 326 | common.error("Deleted {} due to error".format(target_dir)) 327 | except: 328 | raise CCMError("Building C* version {} failed. Attempted to delete {} but failed. This will need to be manually deleted".format(version, target_dir)) 329 | raise e 330 | 331 | 332 | def compile_version(version, target_dir, verbose=False): 333 | # compiling cassandra and the stress tool 334 | logfile = lastlogfilename() 335 | logger = get_logger(logfile) 336 | 337 | common.info("Compiling Cassandra {} ...".format(version)) 338 | logger.info("--- Cassandra Build -------------------\n") 339 | 340 | env = update_java_version(install_dir=target_dir, for_build=True, info_message='Cassandra {} build'.format(version)) 341 | 342 | default_build_properties = os.path.join(common.get_default_path(), 'build.properties.default') 343 | if os.path.exists(default_build_properties): 344 | target_build_properties = os.path.join(target_dir, 'build.properties') 345 | logger.info("Copying %s to %s\n" % (default_build_properties, target_build_properties)) 346 | shutil.copyfile(default_build_properties, target_build_properties) 347 | 348 | try: 349 | # Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543 350 | # Similar patch seen with buildbot 351 | attempt = 0 352 | ret_val = 1 353 | gradlew = os.path.join(target_dir, platform_binary('gradlew')) 354 | if os.path.exists(gradlew): 355 | # todo: move to dse/ 356 | cmd = [gradlew, 'jar'] 357 | else: 358 | mvnw = os.path.join(target_dir, platform_binary('mvnw')) 359 | if os.path.exists(mvnw): 360 | # todo: move to hcd/ 361 | cmd = [mvnw, 'verify', '-DskipTest', '-DskipDocker','-DskipDeb','-DskipRPM','-DskipCqlsh', '-Pdatastax-artifactory'] 362 | else: 363 | # No gradle, use ant 364 | cmd = [platform_binary('ant'), 'jar'] 365 | if get_jdk_version_int(env=env) >= 11: 366 | cmd.append('-Duse.jdk11=true') 367 | while attempt < 3 and ret_val != 0: 368 | if attempt > 0: 369 | logger.info("\n\n`{}` failed. Retry #{}...\n\n".format(' '.join(cmd), attempt)) 370 | process = subprocess.Popen(cmd, cwd=target_dir, env=env, 371 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 372 | ret_val, stdout, stderr = log_info(process, logger) 373 | attempt += 1 374 | if ret_val != 0: 375 | raise CCMError('Error compiling Cassandra. See {logfile} or run ' 376 | '"ccm showlastlog" for details, stdout=\'{stdout}\' stderr=\'{stderr}\''.format( 377 | logfile=logfile, stdout=stdout.decode(), stderr=stderr.decode())) 378 | except OSError as e: 379 | raise CCMError("Error compiling Cassandra. Is ant installed? See %s for details" % logfile) 380 | 381 | stress_dir = os.path.join(target_dir, "tools", "stress") if ( 382 | version >= "0.8.0") else \ 383 | os.path.join(target_dir, "contrib", "stress") 384 | 385 | build_xml = os.path.join(stress_dir, 'build.xml') 386 | if os.path.exists(build_xml): # building stress separately is only necessary pre-1.1 387 | logger.info("\n\n--- cassandra/stress build ------------\n") 388 | try: 389 | # set permissions correctly, seems to not always be the case 390 | stress_bin_dir = os.path.join(stress_dir, 'bin') 391 | for f in os.listdir(stress_bin_dir): 392 | full_path = os.path.join(stress_bin_dir, f) 393 | os.chmod(full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) 394 | 395 | process = subprocess.Popen([platform_binary('ant'), 'build'], cwd=stress_dir, env=env, 396 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 397 | ret_val, _, _ = log_info(process, logger) 398 | if ret_val != 0: 399 | process = subprocess.Popen([platform_binary('ant'), 'stress-build'], cwd=target_dir, env=env, 400 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 401 | ret_val, _, _ = log_info(process, logger) 402 | if ret_val != 0: 403 | raise CCMError("Error compiling Cassandra stress tool. " 404 | "See %s for details (you will still be able to use ccm " 405 | "but not the stress related commands)" % logfile) 406 | except IOError as e: 407 | raise CCMError("Error compiling Cassandra stress tool: %s (you will " 408 | "still be able to use ccm but not the stress related commands)" % str(e)) 409 | 410 | 411 | def directory_name(version): 412 | version = version.replace(':', 'COLON') # handle git branches like 'git:trunk'. 413 | version = version.replace('/', 'SLASH') # handle git branches like 'github:mambocab/trunk'. 414 | return os.path.join(__get_dir(), version) 415 | 416 | 417 | def github_username_and_branch_name(version): 418 | assert version.startswith('github') 419 | return version.split(':', 1)[1].split('/', 1) 420 | 421 | 422 | def github_repo_for_user(username): 423 | return 'https://github.com/{username}/cassandra.git'.format(username=username) 424 | 425 | 426 | def version_directory(version): 427 | dir = directory_name(version) 428 | if os.path.exists(dir): 429 | try: 430 | validate_install_dir(dir) 431 | return dir 432 | except ArgumentError: 433 | rmdirs(dir) 434 | return None 435 | else: 436 | return None 437 | 438 | 439 | def clean_all(): 440 | rmdirs(__get_dir()) 441 | 442 | 443 | def get_tagged_version_numbers(series='stable'): 444 | """Retrieve git tags and find version numbers for a release series 445 | 446 | series - 'stable', 'oldstable', or 'testing'""" 447 | releases = [] 448 | if series == 'testing': 449 | # Testing releases always have a hyphen after the version number: 450 | tag_regex = re.compile('^refs/tags/cassandra-([0-9]+\.[0-9]+\.[0-9]+-.*$)') 451 | else: 452 | # Stable and oldstable releases are just a number: 453 | tag_regex = re.compile('^refs/tags/cassandra-([0-9]+\.[0-9]+\.[0-9]+$)') 454 | 455 | tag_url = urllib.request.urlopen(GITHUB_TAGS) 456 | for ref in (i.get('ref', '') for i in json.loads(tag_url.read())): 457 | m = tag_regex.match(ref) 458 | if m: 459 | releases.append(LooseVersion(m.groups()[0])) 460 | 461 | # Sort by semver: 462 | releases.sort(reverse=True) 463 | 464 | stable_major_version = LooseVersion(str(releases[0].version[0]) + "." + str(releases[0].version[1])) 465 | stable_releases = [r for r in releases if r >= stable_major_version] 466 | oldstable_releases = [r for r in releases if r not in stable_releases] 467 | oldstable_major_version = LooseVersion(str(oldstable_releases[0].version[0]) + "." + str(oldstable_releases[0].version[1])) 468 | oldstable_releases = [r for r in oldstable_releases if r >= oldstable_major_version] 469 | 470 | if series == 'testing': 471 | return [r.vstring for r in releases] 472 | elif series == 'stable': 473 | return [r.vstring for r in stable_releases] 474 | elif series == 'oldstable': 475 | return [r.vstring for r in oldstable_releases] 476 | else: 477 | raise AssertionError("unknown release series: {series}".format(series=series)) 478 | 479 | 480 | def __download(url, target, username=None, password=None, show_progress=False): 481 | if username is not None: 482 | password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() 483 | password_mgr.add_password(None, url, username, password) 484 | handler = urllib.request.HTTPBasicAuthHandler(password_mgr) 485 | opener = urllib.request.build_opener(handler) 486 | urllib.request.install_opener(opener) # pylint: disable=E1121 487 | 488 | u = urllib.request.urlopen(url) 489 | f = open(target, 'wb') 490 | meta = u.info() 491 | file_size = int(meta.get("Content-Length")) 492 | common.info("Downloading {} to {} ({:.3f}MB)".format(url, target, float(file_size) / (1024 * 1024))) 493 | 494 | file_size_dl = 0 495 | block_sz = 8192 496 | status = None 497 | attempts = 0 498 | while file_size_dl < file_size: 499 | buffer = u.read(block_sz) 500 | if not buffer: 501 | attempts = attempts + 1 502 | if attempts >= 5: 503 | raise CCMError("Error downloading file (nothing read after {} attempts, downloded only {} of {} bytes)".format(attempts, file_size_dl, file_size)) 504 | time.sleep(0.5 * attempts) 505 | continue 506 | else: 507 | attempts = 0 508 | 509 | file_size_dl += len(buffer) 510 | f.write(buffer) 511 | if show_progress: 512 | status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size) 513 | status = chr(8) * (len(status) + 1) + status 514 | print_(status, end='') 515 | 516 | f.close() 517 | u.close() 518 | 519 | 520 | def __get_dir(): 521 | repo = os.path.join(get_default_path(), 'repository') 522 | if not os.path.exists(repo): 523 | os.mkdir(repo) 524 | return repo 525 | 526 | 527 | def lastlogfilename(): 528 | return os.path.join(__get_dir(), "ccm-repository.log") 529 | 530 | 531 | def get_logger(log_file): 532 | logger = logging.getLogger('repository') 533 | logger.addHandler(handlers.RotatingFileHandler(log_file, maxBytes=1024 * 1024 * 5, backupCount=5)) 534 | return logger 535 | 536 | 537 | def log_info(process, logger): 538 | stdoutdata, stderrdata = process.communicate() 539 | rc = process.returncode 540 | logger.info(stdoutdata.decode()) 541 | logger.info(stderrdata.decode()) 542 | return rc, stdoutdata, stderrdata 543 | -------------------------------------------------------------------------------- /tests/test_lib.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import os 19 | import sys 20 | import tempfile 21 | import time 22 | from pathlib import Path 23 | 24 | import pytest 25 | import requests 26 | from six import StringIO 27 | 28 | import ccmlib 29 | from ccmlib.cluster import Cluster 30 | from ccmlib.common import _update_java_version, get_supported_jdk_versions_from_dist, get_supported_jdk_versions, get_available_jdk_versions 31 | from ccmlib.node import NodeError 32 | from . import TEST_DIR, ccmtest 33 | from distutils.version import LooseVersion # pylint: disable=import-error, no-name-in-module 34 | 35 | sys.path = [".."] + sys.path 36 | 37 | CLUSTER_PATH = TEST_DIR 38 | 39 | 40 | class TestUpdateJavaVersion(ccmtest.Tester): 41 | env = dict() 42 | temp_dir = None 43 | all_versions = ['2.2', '3.0', '3.1', '3.11', '4.0', '4.1', '5.0', '5.1'] 44 | 45 | # prepare for tests 46 | def setUp(self): 47 | # create a directory in temp location 48 | self.temp_dir = tempfile.TemporaryDirectory() 49 | 50 | # create fake java distribution directories for 7, 8, 11, 17, and 21 in temp directory 51 | for jvm_version in [7, 8, 11, 17, 21]: 52 | path = self.temp_dir.name + '/java{}'.format(jvm_version) 53 | Path(path + '/bin').mkdir(parents=True, exist_ok=True) 54 | # create java executable with a script returning java version 55 | full_version = '{}.0.{}'.format(jvm_version, jvm_version * 2) 56 | build_version = jvm_version * 3 57 | with open(path + '/bin/java', 'w') as f: 58 | f.write('#!/bin/bash\n') 59 | # there must be an only parameter '-version' in the command line 60 | f.write('if [ "$1" != "-version" ]; then\n') 61 | f.write(' exit 1\n') 62 | f.write('fi\n') 63 | f.write('echo \'openjdk version "{}" 2023-04-18\'\n'.format(full_version)) 64 | f.write('echo \'OpenJDK Runtime Environment Temurin-{}+{} (build {}+{})\'\n'.format(full_version, build_version, full_version, build_version)) 65 | f.write('echo \'OpenJDK 64-Bit Server VM Temurin-{}+{} (build {}+{}, mixed mode)\'\n'.format(full_version, build_version, full_version, build_version)) 66 | # make the script executable 67 | os.chmod(path + '/bin/java', 0o755) 68 | self.env['JAVA{}_HOME'.format(jvm_version)] = path 69 | 70 | def _make_cassandra_install_dir(self, git_branch, is_source_dist): 71 | dist_dir = '{}/cassandra/{}'.format(self.temp_dir.name, git_branch) 72 | Path(dist_dir + "/bin").mkdir(parents=True, exist_ok=True) 73 | 74 | if is_source_dist: 75 | if not Path(dist_dir + '/build.xml').exists(): 76 | with open(dist_dir + '/build.xml', 'w') as f: 77 | r = requests.get('https://raw.githubusercontent.com/apache/cassandra/{}/build.xml'.format(git_branch)) 78 | f.write(r.text) 79 | else: 80 | if Path(dist_dir + '/build.xml').exists(): 81 | os.remove(dist_dir + '/build.xml') 82 | 83 | if not Path(dist_dir + '/bin/cassandra.in.sh').exists(): 84 | with open(dist_dir + '/bin/cassandra.in.sh', 'w') as f: 85 | r = requests.get('https://raw.githubusercontent.com/apache/cassandra/{}/bin/cassandra.in.sh'.format(git_branch)) 86 | f.write(r.text) 87 | 88 | return dist_dir 89 | 90 | def _make_env(self, java_home_version=None, java_path_version=None, include_homes=None): 91 | env = dict() 92 | if include_homes: 93 | for v in include_homes: 94 | key = 'JAVA{}_HOME'.format(v) 95 | env[key] = self.env[key] 96 | else: 97 | env = self.env.copy() 98 | if java_home_version is not None: 99 | env['JAVA_HOME'] = self.env['JAVA{}_HOME'.format(java_home_version)] 100 | if java_path_version is not None: 101 | env['PATH'] = self.env['JAVA{}_HOME'.format(java_path_version)] + "/bin:/some/path" 102 | return env 103 | 104 | def _check_env(self, result_env, expected_java_version): 105 | self.assertIn('JAVA_HOME', result_env) 106 | self.assertIn('PATH', result_env) 107 | self.assertEqual(self.env['JAVA{}_HOME'.format(expected_java_version)], result_env['JAVA_HOME']) 108 | self.assertIn('{}/bin'.format(self.env['JAVA{}_HOME'.format(expected_java_version)]), result_env['PATH']) 109 | 110 | def tearDown(self): 111 | self.temp_dir.cleanup() 112 | 113 | def test_get_supported_jdk_versions_from_dist(self): 114 | # we cannot assert anything about trunk except that we can figure out some versions 115 | self.assertIsNotNone(get_supported_jdk_versions_from_dist(self._make_cassandra_install_dir('trunk', False))) 116 | self.assertIsNotNone(get_supported_jdk_versions_from_dist(self._make_cassandra_install_dir('trunk', True))) 117 | 118 | # some commit of Cassandra 5.1 119 | self.assertEqual(get_supported_jdk_versions_from_dist(self._make_cassandra_install_dir('6bae4f76fb043b4c3a3886178b5650b280e9a50b', False)), [11, 17]) 120 | self.assertEqual(get_supported_jdk_versions_from_dist(self._make_cassandra_install_dir('6bae4f76fb043b4c3a3886178b5650b280e9a50b', True)), [11, 17]) 121 | 122 | self.assertIsNone(get_supported_jdk_versions_from_dist(self._make_cassandra_install_dir('cassandra-5.0', False))) 123 | self.assertEqual(get_supported_jdk_versions_from_dist(self._make_cassandra_install_dir('cassandra-5.0', True)), [11, 17]) 124 | 125 | self.assertIsNone(get_supported_jdk_versions_from_dist(self._make_cassandra_install_dir('cassandra-4.1', False))) 126 | self.assertIsNone(get_supported_jdk_versions_from_dist(self._make_cassandra_install_dir('cassandra-4.1', True))) 127 | 128 | def test_supported_jdk_versions(self): 129 | for cassandra_version in [None, '2.2', '3.0', '3.1', '4.0', '4.1']: 130 | self.assertIn(8, get_supported_jdk_versions(cassandra_version, None, False, {'key': 'value'})) 131 | self.assertIn(8, get_supported_jdk_versions(cassandra_version, None, True, {'key': 'value'})) 132 | 133 | for cassandra_version in [None, '2.2', '3.0', '3.1', '4.0', '4.1']: 134 | self.assertIn(8, get_supported_jdk_versions(cassandra_version, None, False, {'key': 'value', 'CASSANDRA_USE_JDK11': 'false'})) 135 | self.assertIn(8, get_supported_jdk_versions(cassandra_version, None, True, {'key': 'value', 'CASSANDRA_USE_JDK11': 'false'})) 136 | 137 | for cassandra_version in ['4.0', '4.1', '5.0', '5.1']: 138 | for usd_jdk_11 in ['true', 'TRUE', 'True', 'on', 'ON', 'On', 'yes', 'YES', 'Yes']: 139 | self.assertNotIn(8, get_supported_jdk_versions(cassandra_version, None, False, {'CASSANDRA_USE_JDK11': usd_jdk_11})) 140 | self.assertNotIn(8, get_supported_jdk_versions(cassandra_version, None, True, {'CASSANDRA_USE_JDK11': usd_jdk_11})) 141 | 142 | for cassandra_version in ['4.0', '4.1', '5.0', '5.1']: 143 | self.assertIn(11, get_supported_jdk_versions(cassandra_version, None, False, {'key': 'value'})) 144 | self.assertIn(11, get_supported_jdk_versions(cassandra_version, None, True, {'key': 'value'})) 145 | 146 | for cassandra_version in [None, '2.2', '3.0', '3.11']: 147 | self.assertNotIn(11, get_supported_jdk_versions(cassandra_version, None, False, {'key': 'value'})) 148 | self.assertNotIn(11, get_supported_jdk_versions(cassandra_version, None, True, {'key': 'value'})) 149 | 150 | for cassandra_version in ['5.0', '5.1']: 151 | self.assertIn(17, get_supported_jdk_versions(cassandra_version, None, False, {'key': 'value'})) 152 | self.assertIn(17, get_supported_jdk_versions(cassandra_version, None, True, {'key': 'value'})) 153 | 154 | for cassandra_version in [None, '2.2', '3.0', '3.11', '4.0', '4.1']: 155 | self.assertNotIn(17, get_supported_jdk_versions(cassandra_version, None, False, {'key': 'value'})) 156 | self.assertNotIn(17, get_supported_jdk_versions(cassandra_version, None, True, {'key': 'value'})) 157 | 158 | def test_get_available_jdk_versions(self): 159 | self.assertDictEqual(get_available_jdk_versions(self._make_env()), {7: 'JAVA7_HOME', 8: 'JAVA8_HOME', 11: 'JAVA11_HOME', 17: 'JAVA17_HOME', 21: 'JAVA21_HOME'}) 160 | self.assertDictEqual(get_available_jdk_versions(self._make_env(java_home_version=8, include_homes=[11, 17])), {8: 'JAVA_HOME', 11: 'JAVA11_HOME', 17: 'JAVA17_HOME'}) 161 | 162 | def _test_java_selection(self, expected_version, path_version, home_version, explicit_version, cassandra_versions, available_homes=None): 163 | for cassandra_version in cassandra_versions: 164 | result_env = _update_java_version(current_java_version=path_version, current_java_home_version=home_version, 165 | jvm_version=explicit_version, install_dir=None, cassandra_version=LooseVersion(cassandra_version), 166 | env=self._make_env(java_home_version=home_version, java_path_version=path_version, include_homes=available_homes), 167 | for_build=True, info_message='test_java_selection_{}'.format(cassandra_version), os_env={'key': 'value'}) 168 | self._check_env(result_env, expected_version) 169 | if expected_version >= 11 and '4.0' <= cassandra_version < '5.0': 170 | self.assertEqual(result_env['CASSANDRA_USE_JDK11'], 'true') 171 | else: 172 | self.assertNotIn('CASSANDRA_USE_JDK11', result_env) 173 | 174 | def _test_java_selection_fail(self, expected_failure_regexp, path_version, home_version, explicit_version, cassandra_versions, available_homes=None): 175 | for cassandra_version in cassandra_versions: 176 | self.assertRaisesRegex(RuntimeError, expected_failure_regexp, _update_java_version, 177 | path_version, home_version, explicit_version, None, LooseVersion(cassandra_version), 178 | self._make_env(java_home_version=home_version, java_path_version=path_version, include_homes=available_homes), 179 | True, 'test_java_selection_fail_{}'.format(cassandra_version), {'key': 'value'}) 180 | 181 | def test_update_java_version(self): 182 | # use the highest supported version if there is no current Java command available 183 | self._test_java_selection(8, None, None, None, ['2.2', '3.0', '3.11']) 184 | self._test_java_selection(11, None, None, None, ['4.0', '4.1']) 185 | self._test_java_selection(17, None, None, None, ['5.0', '5.1']) 186 | 187 | self._test_java_selection(8, None, None, None, ['4.0', '4.1'], available_homes=[8]) 188 | self._test_java_selection(11, None, None, None, ['4.0', '4.1'], available_homes=[8, 11]) 189 | self._test_java_selection(11, None, None, None, ['4.0', '4.1'], available_homes=[8, 11]) 190 | 191 | # use the current Java version if it is supported, otherwise use the closest possible Java version 192 | # current is 8 193 | self._test_java_selection(8, 8, 8, None, ['2.2', '3.0', '3.11', '4.0', '4.1']) 194 | self._test_java_selection(11, 8, 8, None, ['5.0', '5.1']) 195 | # current is 11 196 | self._test_java_selection(8, 11, 11, None, ['2.2', '3.0', '3.11']) 197 | self._test_java_selection(11, 11, 11, None, ['4.0', '4.1', '5.0', '5.1']) 198 | # current is 17 199 | self._test_java_selection(8, 17, 17, None, ['2.2', '3.0', '3.11']) 200 | self._test_java_selection(11, 17, 17, None, ['4.0', '4.1']) 201 | self._test_java_selection(17, 17, 17, None, ['5.0', '5.1']) 202 | 203 | # same as above, but now there is only JAVA_HOME defined and no Java command on the PATH 204 | self._test_java_selection(8, None, 8, None, ['2.2', '3.0', '3.11', '4.0', '4.1']) 205 | self._test_java_selection(11, None, 8, None, ['5.0', '5.1']) 206 | self._test_java_selection(8, None, 11, None, ['2.2', '3.0', '3.11']) 207 | self._test_java_selection(11, None, 11, None, ['4.0', '4.1', '5.0', '5.1']) 208 | self._test_java_selection(8, None, 17, None, ['2.2', '3.0', '3.11']) 209 | self._test_java_selection(11, None, 17, None, ['4.0', '4.1']) 210 | self._test_java_selection(17, None, 17, None, ['5.0', '5.1']) 211 | 212 | # set explicit version should override everything 213 | self._test_java_selection(21, None, None, 21, self.all_versions) 214 | self._test_java_selection(21, 8, 8, 21, self.all_versions) 215 | self._test_java_selection(21, 11, 11, 21, self.all_versions) 216 | self._test_java_selection(21, 17, 17, 21, self.all_versions) 217 | 218 | # fail if the required Java version is not available 219 | self._test_java_selection_fail("Cannot find any Java distribution for the current invocation", None, None, None, self.all_versions, available_homes=[21]) 220 | self._test_java_selection_fail("The explicitly requested Java version 11 is not available in the current env", None, None, 11, self.all_versions, available_homes=[8]) 221 | 222 | # fail if home and path are inconsistent 223 | self._test_java_selection_fail("The version of java available on PATH 8 does not match the Java version of the distribution provided via JAVA_HOME 11", path_version=8, home_version=11, explicit_version=None, cassandra_versions=self.all_versions) 224 | self._test_java_selection_fail("JAVA_HOME must be defined if java command is available on the PATH", path_version=8, home_version=None, explicit_version=None, cassandra_versions=self.all_versions) 225 | 226 | @pytest.mark.skip(reason="this test is starting nodes - it needs to be refactored to use mocks instead") 227 | class TestCCMLib(ccmtest.Tester): 228 | def test2(self): 229 | self.cluster = Cluster(CLUSTER_PATH, "test2", version='git:trunk') 230 | self.cluster.populate(2) 231 | self.cluster.start() 232 | 233 | self.cluster.set_log_level("ERROR") 234 | 235 | class FakeNode: 236 | name = "non-existing node" 237 | 238 | self.cluster.remove(FakeNode()) 239 | node1 = self.cluster.nodelist()[0] 240 | self.cluster.remove(node1) 241 | self.cluster.show(True) 242 | self.cluster.show(False) 243 | 244 | self.cluster.compact() 245 | self.cluster.drain() 246 | self.cluster.stop() 247 | 248 | def test3(self): 249 | self.cluster = Cluster(CLUSTER_PATH, "test3", version='git:trunk') 250 | self.cluster.populate(2) 251 | self.cluster.start() 252 | node1 = self.cluster.nodelist()[0] 253 | self.cluster.stress(['write', 'n=100', 'no-warmup', '-rate', 'threads=4']) 254 | self.cluster.stress(['write', 'n=100', 'no-warmup', '-rate', 'threads=4', '-node', node1.ip_addr]) 255 | 256 | self.cluster.clear() 257 | self.cluster.stop() 258 | 259 | def test_node_start_with_non_default_timeout(self): 260 | self.cluster = Cluster(CLUSTER_PATH, "nodestarttimeout", cassandra_version='git:trunk') 261 | self.cluster.populate(1) 262 | node = self.cluster.nodelist()[0] 263 | 264 | try: 265 | node.start(wait_for_binary_proto=0) 266 | self.fail("timeout expected with 0s startup timeout") 267 | except ccmlib.node.TimeoutError: 268 | pass 269 | finally: 270 | self.cluster.clear() 271 | self.cluster.stop() 272 | 273 | def test_fast_error_on_cluster_start_failure(self): 274 | self.cluster = Cluster(CLUSTER_PATH, 'clusterfaststop', cassandra_version='git:trunk') 275 | self.cluster.populate(1) 276 | self.cluster.set_configuration_options({'invalid_option': 0}) 277 | self.check_log_errors = False 278 | node = self.cluster.nodelist()[0] 279 | start_time = time.time() 280 | try: 281 | self.cluster.start(use_vnodes=True) 282 | self.assertFalse(node.is_running()) 283 | self.assertLess(time.time() - start_time, 30) 284 | except NodeError: 285 | self.assertLess(time.time() - start_time, 30) 286 | finally: 287 | self.cluster.clear() 288 | self.cluster.stop() 289 | 290 | def test_fast_error_on_node_start_failure(self): 291 | self.cluster = Cluster(CLUSTER_PATH, 'nodefaststop', cassandra_version='git:trunk') 292 | self.cluster.populate(1) 293 | self.cluster.set_configuration_options({'invalid_option': 0}) 294 | self.check_log_errors = False 295 | node = self.cluster.nodelist()[0] 296 | start_time = time.time() 297 | try: 298 | node.start(wait_for_binary_proto=45) 299 | self.assertFalse(node.is_running()) 300 | self.assertLess(time.time() - start_time, 30) 301 | except NodeError: 302 | self.assertLess(time.time() - start_time, 30) 303 | finally: 304 | self.cluster.clear() 305 | self.cluster.stop() 306 | 307 | def test_dc_mandatory_on_multidc(self): 308 | self.cluster = Cluster(CLUSTER_PATH, "mandatorydc", cassandra_version='git:trunk') 309 | self.cluster.populate([1, 1]) 310 | 311 | node3 = self.cluster.create_node(name='node3', 312 | auto_bootstrap=True, 313 | thrift_interface=('127.0.0.3', 9160), 314 | storage_interface=('127.0.0.3', 7000), 315 | jmx_port='7300', 316 | remote_debug_port='0', 317 | initial_token=None, 318 | binary_interface=('127.0.0.3', 9042)) 319 | with self.assertRaisesRegexp(ccmlib.common.ArgumentError, 'Please specify the DC this node should be added to'): 320 | self.cluster.add(node3, is_seed=False) 321 | 322 | # TODO remove this unused class 323 | class TestRunCqlsh(ccmtest.Tester): 324 | 325 | def setUp(self): 326 | '''Create a cluster for cqlsh tests. Assumes that ccmtest.Tester's 327 | teardown() method will safely stop and remove self.cluster.''' 328 | self.cluster = Cluster(CLUSTER_PATH, "run_cqlsh", 329 | cassandra_version='git:trunk') 330 | self.cluster.populate(1).start(wait_for_binary_proto=True) 331 | [self.node] = self.cluster.nodelist() 332 | 333 | def run_cqlsh_printing(self, return_output, show_output): 334 | '''Parameterized test. Runs run_cqlsh with options to print the output 335 | and to return it as a string, or with these options combined, depending 336 | on the values of the arguments.''' 337 | # redirect run_cqlsh's stdout to a string buffer 338 | old_stdout, sys.stdout = sys.stdout, StringIO() 339 | 340 | rv = self.node.run_cqlsh('DESCRIBE keyspaces;', 341 | return_output=return_output, 342 | show_output=show_output) 343 | 344 | # put stdout back where it belongs and get the built string value 345 | sys.stdout, printed_output = old_stdout, sys.stdout.getvalue() 346 | 347 | if return_output: 348 | # we should see names of system keyspaces 349 | self.assertIn('system', rv[0]) 350 | # stderr should be empty 351 | self.assertEqual('', rv[1]) 352 | else: 353 | # implicitly-returned None 354 | self.assertEqual(rv, None) 355 | 356 | if show_output: 357 | self.assertIn('system', printed_output) 358 | else: 359 | # nothing should be printed if (not show_output) 360 | self.assertEqual(printed_output, '') 361 | 362 | if return_output and show_output: 363 | self.assertEqual(printed_output, rv[0]) 364 | 365 | 366 | class TestNodeLoad(ccmtest.Tester): 367 | 368 | def test_rejects_multiple_load_lines(self): 369 | info = 'Load : 699 KiB\nLoad : 35 GiB' 370 | with self.assertRaises(RuntimeError): 371 | ccmlib.node._get_load_from_info_output(info) 372 | 373 | info = 'Load : 699 KB\nLoad : 35 GB' 374 | with self.assertRaises(RuntimeError): 375 | ccmlib.node._get_load_from_info_output(info) 376 | 377 | def test_rejects_unexpected_units(self): 378 | infos = ['Load : 200 PiB', 'Load : 200 PB', 'Load : 12 Parsecs'] 379 | 380 | for info in infos: 381 | with self.assertRaises(RuntimeError): 382 | ccmlib.node._get_load_from_info_output(info) 383 | 384 | def test_gets_correct_value(self): 385 | info_value = [('Load : 328.45 KiB', 328.45), 386 | ('Load : 328.45 KB', 328.45), 387 | ('Load : 295.72 MiB', 295.72 * 1024), 388 | ('Load : 295.72 MB', 295.72 * 1024), 389 | ('Load : 183.79 GiB', 183.79 * 1024 * 1024), 390 | ('Load : 183.79 GB', 183.79 * 1024 * 1024), 391 | ('Load : 82.333 TiB', 82.333 * 1024 * 1024 * 1024), 392 | ('Load : 82.333 TB', 82.333 * 1024 * 1024 * 1024)] 393 | 394 | for info, value in info_value: 395 | self.assertEqual(ccmlib.node._get_load_from_info_output(info), 396 | value) 397 | 398 | def test_with_full_info_output(self): 399 | data = ('ID : 82800bf3-8c1a-4355-9b72-e19aa61d9fba\n' 400 | 'Gossip active : true\n' 401 | 'Thrift active : true\n' 402 | 'Native Transport active: true\n' 403 | 'Load : 247.59 MiB\n' 404 | 'Generation No : 1426190195\n' 405 | 'Uptime (seconds) : 526\n' 406 | 'Heap Memory (MB) : 222.83 / 495.00\n' 407 | 'Off Heap Memory (MB) : 1.16\n' 408 | 'Data Center : dc1\n' 409 | 'Rack : r1\n' 410 | 'Exceptions : 0\n' 411 | 'Key Cache : entries 41, size 3.16 KB, capacity 24 MB, 19 hits, 59 requests, 0.322 recent hit rate, 14400 save period in seconds\n' 412 | 'Row Cache : entries 0, size 0 bytes, capacity 0 bytes, 0 hits, 0 requests, NaN recent hit rate, 0 save period in seconds\n' 413 | 'Counter Cache : entries 0, size 0 bytes, capacity 12 MB, 0 hits, 0 requests, NaN recent hit rate, 7200 save period in seconds\n' 414 | 'Token : -9223372036854775808\n') 415 | self.assertEqual(ccmlib.node._get_load_from_info_output(data), 416 | 247.59 * 1024) 417 | 418 | data = ('ID : 82800bf3-8c1a-4355-9b72-e19aa61d9fba\n' 419 | 'Gossip active : true\n' 420 | 'Thrift active : true\n' 421 | 'Native Transport active: true\n' 422 | 'Load : 247.59 MB\n' 423 | 'Generation No : 1426190195\n' 424 | 'Uptime (seconds) : 526\n' 425 | 'Heap Memory (MB) : 222.83 / 495.00\n' 426 | 'Off Heap Memory (MB) : 1.16\n' 427 | 'Data Center : dc1\n' 428 | 'Rack : r1\n' 429 | 'Exceptions : 0\n' 430 | 'Key Cache : entries 41, size 3.16 KB, capacity 24 MB, 19 hits, 59 requests, 0.322 recent hit rate, 14400 save period in seconds\n' 431 | 'Row Cache : entries 0, size 0 bytes, capacity 0 bytes, 0 hits, 0 requests, NaN recent hit rate, 0 save period in seconds\n' 432 | 'Counter Cache : entries 0, size 0 bytes, capacity 12 MB, 0 hits, 0 requests, NaN recent hit rate, 7200 save period in seconds\n' 433 | 'Token : -9223372036854775808\n') 434 | self.assertEqual(ccmlib.node._get_load_from_info_output(data), 435 | 247.59 * 1024) 436 | 437 | 438 | class TestErrorLogGrepping(ccmtest.Tester): 439 | 440 | def assertGreppedLog(self, log, grepped_log): 441 | self.assertEqual(ccmlib.node._grep_log_for_errors(log), grepped_log) 442 | 443 | def test_basic_error_message(self): 444 | err = 'ERROR: You messed up' 445 | self.assertGreppedLog(err, [[err]]) 446 | 447 | def test_error_message_with_timestamp(self): 448 | err = '2015-05-12 14:12:12,720 ERROR: You messed up' 449 | self.assertGreppedLog(err, [[err]]) 450 | 451 | def test_filter_debug_lines(self): 452 | err = 'DEBUG: harmless warning message\n' 453 | self.assertGreppedLog(err, []) 454 | 455 | def test_ignore_empty_lines(self): 456 | err = ('\n' 457 | 'ERROR: Node unavailable') 458 | self.assertGreppedLog(err, [['ERROR: Node unavailable']]) 459 | 460 | def test_ignore_debug_lines_containing_error(self): 461 | err = 'DEBUG: another process raised: ERROR: abandon hope!\n' 462 | self.assertGreppedLog(err, []) 463 | 464 | def test_coalesces_stack_trace_lines(self): 465 | err = ('ERROR: You have made a terrible mistake\n' 466 | ' And here are more details on what you did\n' 467 | 'saints preserve us') 468 | self.assertGreppedLog(err, 469 | [['ERROR: You have made a terrible mistake', 470 | ' And here are more details on what you did', 471 | 'saints preserve us']]) 472 | 473 | def test_multiple_errors(self): 474 | err = ('ERROR: You have made a terrible mistake\n' 475 | ' And here are more details on what you did\n' 476 | 'INFO: Node joined ring\n' 477 | 'ERROR: not again!') 478 | self.assertGreppedLog(err, 479 | [['ERROR: You have made a terrible mistake', 480 | ' And here are more details on what you did'], 481 | ['ERROR: not again!']]) 482 | 483 | def test_consecutive_errors(self): 484 | err = ('ERROR: You have made a terrible mistake\n' 485 | ' And here are more details on what you did\n' 486 | 'INFO: Node joined ring\n' 487 | 'ERROR: not again!\n' 488 | ' And, yup, here is some more details\n' 489 | 'ERROR: ugh, and a third one!') 490 | self.assertGreppedLog(err, 491 | [['ERROR: You have made a terrible mistake', 492 | ' And here are more details on what you did'], 493 | ['ERROR: not again!', 494 | ' And, yup, here is some more details'], 495 | ['ERROR: ugh, and a third one!']]) 496 | 497 | def test_does_not_coalesce_info_lines(self): 498 | err = ('ERROR: You have made a terrible mistake\n' 499 | ' 2015-05-12 14:12:12,720 INFO: why would you ever do that\n') 500 | self.assertGreppedLog(err, [['ERROR: You have made a terrible mistake']]) 501 | 502 | def test_finds_exceptions_logged_as_warn(self): 503 | line1 = ('WARN [ReadStage-2] 2017-03-20 13:29:39,165 AbstractLocalAwareExecutorService.java:167 - ' 504 | 'Uncaught exception on thread Thread[ReadStage-2,5,main]: {} java.lang.AssertionError: Lower bound ' 505 | '[INCL_START_BOUND(HistLoss, -9223372036854775808, -9223372036854775808) ]is bigger than ' 506 | 'first returned value') 507 | 508 | line2 = ('WARN [MessagingService-Incoming-/IP] 2017-05-26 19:27:11,523 IncomingTcpConnection.java:101 - ' 509 | 'UnknownColumnFamilyException reading from socket; closing org.apache.cassandra.db.' 510 | 'UnknownColumnFamilyException: Couldnt find table for cfId 922b7940-3a65-11e7-adf3-a3ff55d9bcf1. ' 511 | 'If a table was just created, this is likely due to the schema not being fully propagated. ' 512 | 'Please wait for schema agreement on table creation.') 513 | 514 | line3 = 'WARN oh no there was an error, it failed, with a failure' # dont care for this one 515 | 516 | line4 = 'WARN there was an exception!!!' 517 | 518 | err = '\n'.join([line1, line2, line3, line4]) 519 | self.assertGreppedLog(err, [[line1], [line2], [line4]]) 520 | 521 | 522 | class TestCCMCLI(ccmtest.Tester): 523 | def test_help_message_with_paramiko(self): 524 | self._test_help_message(True) 525 | 526 | def test_help_message_without_paramiko(self): 527 | self._test_help_message(False) 528 | 529 | @staticmethod 530 | def _test_help_message(paramiko: bool): 531 | import ccmlib.remote 532 | before = ccmlib.remote.PARAMIKO_IS_AVAILABLE 533 | try: 534 | ccmlib.remote.PARAMIKO_IS_AVAILABLE = paramiko 535 | from ccmlib.cmds.common import get_command 536 | cmd = get_command('cluster', 'create') 537 | cmd.get_parser().print_help() 538 | finally: 539 | ccmlib.remote.PARAMIKO_IS_AVAILABLE = before 540 | --------------------------------------------------------------------------------