├── ceph_salt ├── validate │ ├── __init__.py │ ├── salt_minion.py │ ├── salt_master.py │ └── config.py ├── params_helper.py ├── logging_utils.py ├── exceptions.py ├── terminal_utils.py ├── __init__.py ├── salt_event.py └── core.py ├── ceph-salt-formula ├── salt │ ├── ceph-salt │ │ ├── init.sls │ │ ├── apply │ │ │ ├── files │ │ │ │ ├── managed-header.txt.j2 │ │ │ │ ├── 90-ceph-salt.conf.j2 │ │ │ │ ├── chrony_sync_clock.sh │ │ │ │ ├── latency.conf.j2 │ │ │ │ ├── chrony.conf.j2 │ │ │ │ ├── throughput.conf.j2 │ │ │ │ └── registries.conf.j2 │ │ │ ├── apply-end.sls │ │ │ ├── software.sls │ │ │ ├── init.sls │ │ │ ├── sysctl.sls │ │ │ ├── cephorch.sls │ │ │ ├── cephtools.sls │ │ │ ├── apparmor.sls │ │ │ ├── tuned-latency.sls │ │ │ ├── tuned-throughput.sls │ │ │ ├── ceph-admin.sls │ │ │ ├── tuned-off.sls │ │ │ ├── time-prep.sls │ │ │ ├── container.sls │ │ │ ├── cephconfigure.sls │ │ │ ├── time-sync.sls │ │ │ └── cephbootstrap.sls │ │ ├── stop │ │ │ ├── stop-end.sls │ │ │ ├── init.sls │ │ │ └── stop.sls │ │ ├── reboot │ │ │ ├── reboot-end.sls │ │ │ ├── init.sls │ │ │ └── reboot.sls │ │ ├── update │ │ │ ├── update-end.sls │ │ │ ├── init.sls │ │ │ └── update.sls │ │ ├── files │ │ │ └── registry-login-json.j2 │ │ ├── common │ │ │ ├── sshkey-cleanup.sls │ │ │ ├── orch-host-label.sls │ │ │ ├── install-cephadm.sls │ │ │ └── sshkey.sls │ │ ├── purge │ │ │ └── init.sls │ │ └── reset.sls │ ├── macros.yml │ ├── _modules │ │ ├── ceph_orch.py │ │ ├── ceph_salt.py │ │ └── multi.py │ └── _states │ │ ├── ceph_salt.py │ │ └── ceph_orch.py └── metadata │ ├── metadata.yml │ └── pillar.example ├── _images ├── architecture.odg └── architecture.png ├── .github ├── READ_THIS └── workflows │ ├── testing.yml │ ├── linting.yml │ └── tox.yml ├── requirements.txt ├── .gitignore ├── tox.ini ├── setup.py ├── LICENSE ├── Jenkinsfile.integration ├── HOWTORELEASE.md ├── tests ├── test_grains_manager.py ├── test_validate_salt_master.py ├── test_ssh_manager.py ├── test_pillar_manager.py ├── __init__.py ├── test_salt_event.py └── test_execute.py ├── setup.cfg ├── ceph-salt.spec ├── README.md ├── ceph-salt.8 ├── .pylintrc └── CHANGELOG.md /ceph_salt/validate/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/init.sls: -------------------------------------------------------------------------------- 1 | include: 2 | - .apply 3 | -------------------------------------------------------------------------------- /_images/architecture.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ceph/ceph-salt/master/_images/architecture.odg -------------------------------------------------------------------------------- /_images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ceph/ceph-salt/master/_images/architecture.png -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/files/managed-header.txt.j2: -------------------------------------------------------------------------------- 1 | This file is managed by ceph-salt. -------------------------------------------------------------------------------- /ceph-salt-formula/metadata/metadata.yml: -------------------------------------------------------------------------------- 1 | description: Ceph cluster deployment 2 | group: general_system_configuration 3 | -------------------------------------------------------------------------------- /.github/READ_THIS: -------------------------------------------------------------------------------- 1 | GitHub Actions and Workflows are not being used at the moment 2 | (Travis and tox are being used instead). 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | . 2 | mock==3.0.5 3 | pycodestyle==2.5.0 4 | pyfakefs==3.7 5 | pylint==2.4.4 6 | pytest==5.3.1 7 | pytest-cov==2.8.1 8 | pytest-runner==5.2 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv* 2 | virtualenv* 3 | .vscode 4 | **/__pycache__ 5 | *.log 6 | **.pyc 7 | *.egg-info 8 | .eggs 9 | .coverage 10 | .pytest_cache 11 | .tox 12 | .idea 13 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/files/90-ceph-salt.conf.j2: -------------------------------------------------------------------------------- 1 | # {% include "ceph-salt/apply/files/managed-header.txt.j2" ignore missing %} 2 | fs.aio-max-nr = 1048576 3 | kernel.pid_max = 4194304 4 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/stop/stop-end.sls: -------------------------------------------------------------------------------- 1 | include: 2 | - ..common.sshkey-cleanup 3 | 4 | set stopped: 5 | grains.present: 6 | - name: ceph-salt:execution:stopped 7 | - value: True 8 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/reboot/reboot-end.sls: -------------------------------------------------------------------------------- 1 | include: 2 | - ..common.sshkey-cleanup 3 | 4 | set rebooted: 5 | grains.present: 6 | - name: ceph-salt:execution:rebooted 7 | - value: True 8 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/apply-end.sls: -------------------------------------------------------------------------------- 1 | include: 2 | - ..common.sshkey-cleanup 3 | 4 | remove ceph-salt-registry-json: 5 | file.absent: 6 | - name: /tmp/ceph-salt-registry-json 7 | - failhard: True 8 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/update/update-end.sls: -------------------------------------------------------------------------------- 1 | include: 2 | - ..common.sshkey-cleanup 3 | - ..common.orch-host-label 4 | 5 | set updated: 6 | grains.present: 7 | - name: ceph-salt:execution:updated 8 | - value: True 9 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/files/registry-login-json.j2: -------------------------------------------------------------------------------- 1 | {% set auth = pillar['ceph-salt'].get('container', {}).get('auth', {}) %} 2 | { 3 | "url": "{{ auth.get('registry') }}", 4 | "username": "{{ auth.get('username') }}", 5 | "password": "{{ auth.get('password') }}" 6 | } 7 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/stop/init.sls: -------------------------------------------------------------------------------- 1 | {% if grains['id'] in pillar['ceph-salt']['minions']['all'] %} 2 | 3 | include: 4 | - ..reset 5 | - ..common.sshkey 6 | - .stop 7 | - .stop-end 8 | 9 | {% else %} 10 | 11 | nothing to do in this node: 12 | test.nop 13 | 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/reboot/init.sls: -------------------------------------------------------------------------------- 1 | {% if grains['id'] in pillar['ceph-salt']['minions']['all'] %} 2 | 3 | include: 4 | - ..reset 5 | - ..common.sshkey 6 | - .reboot 7 | - .reboot-end 8 | 9 | {% else %} 10 | 11 | nothing to do in this node: 12 | test.nop 13 | 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/update/init.sls: -------------------------------------------------------------------------------- 1 | {% if grains['id'] in pillar['ceph-salt']['minions']['all'] %} 2 | 3 | include: 4 | - ..common.install-cephadm 5 | - ..reset 6 | - .update 7 | - ..common.sshkey 8 | - .update-end 9 | 10 | {% else %} 11 | 12 | nothing to do in this node: 13 | test.nop 14 | 15 | {% endif %} 16 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/files/chrony_sync_clock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | # address https://github.com/ceph/ceph-salt/issues/238 4 | # by waiting up to 5 minutes for chrony to see its sources 5 | for trywait in "$(seq 0 9)" ; do 6 | chronyc 'burst 4/4' && break || true 7 | sleep 30 8 | done 9 | sleep 15 10 | chronyc makestep 11 | chronyc waitsync 60 0.04 12 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/common/sshkey-cleanup.sls: -------------------------------------------------------------------------------- 1 | {% if 'admin' not in grains['ceph-salt']['roles'] %} 2 | 3 | remove ceph-salt-ssh-id_rsa: 4 | file.absent: 5 | - name: {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa 6 | - failhard: True 7 | 8 | remove ceph-salt-ssh-id_rsa.pub: 9 | file.absent: 10 | - name: {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa.pub 11 | - failhard: True 12 | 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py3,lint 3 | minversion = 2.0 4 | skipsdist = True 5 | 6 | [testenv] 7 | usedevelop = True 8 | install_command = pip install {opts} {packages} 9 | deps = 10 | -r{toxinidir}/requirements.txt 11 | commands = pytest {posargs:--cov -vv} 12 | 13 | [testenv:lint] 14 | basepython = python3 15 | deps = {[testenv]deps} 16 | commands = 17 | pylint ceph_salt 18 | pycodestyle ceph_salt 19 | pylint tests 20 | pycodestyle tests 21 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/files/latency.conf.j2: -------------------------------------------------------------------------------- 1 | # {% include "ceph-salt/apply/files/managed-header.txt.j2" ignore missing %} 2 | [main] 3 | include=network-latency 4 | dynamic_tuning = 0 5 | 6 | [disk] 7 | # hdparm -S 0 /dev/ 8 | # Sets the timeout to 0 - A value of zero means "timeouts are disabled": the 9 | # device will not automatically enter standby mode. 10 | spindown = 0 11 | # Disable dynamic tuning for disk settings 12 | dynamic = 0 13 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/purge/init.sls: -------------------------------------------------------------------------------- 1 | {% if grains['id'] in pillar['ceph-salt']['minions']['all'] %} 2 | 3 | check safety: 4 | ceph_salt.check_safety: 5 | - failhard: True 6 | 7 | check fsid: 8 | ceph_salt.check_fsid: 9 | - formula: ceph-salt.purge 10 | - failhard: True 11 | 12 | remove clusters: 13 | ceph_orch.rm_clusters: 14 | - failhard: True 15 | 16 | {% else %} 17 | 18 | nothing to do in this node: 19 | test.nop 20 | 21 | {% endif %} 22 | -------------------------------------------------------------------------------- /ceph-salt-formula/metadata/pillar.example: -------------------------------------------------------------------------------- 1 | ceph-salt: 2 | minions: 3 | - 4 | - 5 | # ... 6 | bootstrap_minion: 7 | ssh: 8 | private_key: | 9 | 10 | 11 | # ... 12 | 13 | public_key: 14 | time_server: 15 | enabled: True 16 | host_server: 17 | external_time_servers: 18 | - 19 | - 20 | # ... 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import re 4 | from setuptools import setup 5 | 6 | 7 | def get_version_from_spec(): 8 | this_dir = os.path.dirname(__file__) 9 | with open(os.path.join(this_dir, 'ceph-salt.spec'), 'r') as file: 10 | while True: 11 | line = file.readline() 12 | if not line: 13 | return 'unknown' 14 | ver_match = re.match(r'^Version:\s+(\d.*)', line) 15 | if ver_match: 16 | return ver_match[1] 17 | 18 | 19 | setup( 20 | version=get_version_from_spec(), 21 | ) 22 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/software.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {{ macros.begin_stage('Install required packages') }} 4 | 5 | install required packages: 6 | pkg.installed: 7 | - pkgs: 8 | - catatonit 9 | - hostname 10 | - iperf 11 | - iputils 12 | - lsof 13 | - podman 14 | - rsync 15 | - failhard: True 16 | 17 | /var/log/journal: 18 | file.directory: 19 | - user: root 20 | - group: root 21 | - mode: '0755' 22 | - makedirs: True 23 | - failhard: True 24 | 25 | {{ macros.end_stage('Install required packages') }} 26 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/reset.sls: -------------------------------------------------------------------------------- 1 | reset failure: 2 | grains.present: 3 | - name: ceph-salt:execution:failed 4 | - value: False 5 | 6 | reset updated: 7 | grains.present: 8 | - name: ceph-salt:execution:updated 9 | - value: False 10 | 11 | reset rebooted: 12 | grains.present: 13 | - name: ceph-salt:execution:rebooted 14 | - value: False 15 | 16 | reset stopped: 17 | grains.present: 18 | - name: ceph-salt:execution:stopped 19 | - value: False 20 | 21 | reset stoptimeserversyncedped: 22 | grains.present: 23 | - name: ceph-salt:execution:timeserversynced 24 | - value: False 25 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/init.sls: -------------------------------------------------------------------------------- 1 | {% if grains['id'] in pillar['ceph-salt']['minions']['all'] %} 2 | 3 | include: 4 | - ..common.install-cephadm 5 | - ..reset 6 | - ..common.sshkey 7 | - .sysctl 8 | - .tuned-off 9 | - .tuned-latency 10 | - .tuned-throughput 11 | - .software 12 | - .container 13 | - .apparmor 14 | - .time-prep 15 | - .time-sync 16 | - .cephtools 17 | - .cephbootstrap 18 | - .cephconfigure 19 | - .cephorch 20 | - .ceph-admin 21 | - .apply-end 22 | 23 | {% else %} 24 | 25 | nothing to do in this node: 26 | test.nop 27 | 28 | {% endif %} 29 | -------------------------------------------------------------------------------- /.github/workflows/testing.yml: -------------------------------------------------------------------------------- 1 | name: Testing 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-20.04 9 | strategy: 10 | matrix: 11 | py: 12 | - 3.6 13 | - 3.8 14 | 15 | steps: 16 | - uses: actions/checkout@v1 17 | - name: Set up Python ${{ matrix.py }} 18 | uses: actions/setup-python@v1 19 | with: 20 | python-version: ${{ matrix.py }} 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install -r requirements.txt 25 | - name: Test with pytest 26 | run: | 27 | pytest -s --cov -vv 28 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/sysctl.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {% if 'cephadm' in grains['ceph-salt']['roles'] %} 4 | 5 | {{ macros.begin_stage('Configure sysctl') }} 6 | 7 | /usr/lib/sysctl.d/90-ceph-salt.conf: 8 | file.managed: 9 | - source: 10 | - salt://ceph-salt/apply/files/90-ceph-salt.conf.j2 11 | - template: jinja 12 | - user: root 13 | - group: root 14 | - mode: '0644' 15 | - makedirs: True 16 | - backup: minion 17 | - failhard: True 18 | 19 | reload sysctl: 20 | cmd.run: 21 | - name: "sysctl --system" 22 | 23 | {{ macros.end_stage('Configure sysctl') }} 24 | 25 | {% endif %} 26 | 27 | sysctl: 28 | test.nop 29 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/common/orch-host-label.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {% set my_hostname = salt['ceph_salt.hostname']() %} 4 | 5 | {% if 'admin' in grains['ceph-salt']['roles'] %} 6 | 7 | {% if pillar['ceph-salt'].get ('execution', {}).get('deployed') != False %} 8 | {{ macros.begin_stage('Add host labels to ceph orchestrator') }} 9 | add _admin host label to ceph orch: 10 | ceph_orch.add_host_label: 11 | - host: {{ my_hostname }} 12 | - label: {{ '_admin' }} 13 | - failhard: True 14 | {{ macros.end_stage('Add host labels to ceph orchestrator') }} 15 | {% endif %} 16 | 17 | {% else %} 18 | 19 | not an admin role: 20 | test.nop 21 | 22 | {% endif %} 23 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/cephorch.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {% if 'cephadm' in grains['ceph-salt']['roles'] %} 4 | 5 | {% set my_hostname = salt['ceph_salt.hostname']() %} 6 | {% set my_ipaddr = salt['ceph_salt.ip_address']() %} 7 | {% set is_admin = 'admin' in grains['ceph-salt']['roles'] %} 8 | 9 | {{ macros.begin_stage('Add host to ceph orchestrator') }} 10 | add host to ceph orch: 11 | ceph_orch.add_host: 12 | - host: {{ my_hostname }} 13 | - ipaddr: {{ my_ipaddr }} 14 | - is_admin: {{ is_admin }} 15 | - failhard: True 16 | {{ macros.end_stage('Add host to ceph orchestrator') }} 17 | 18 | {% else %} 19 | 20 | no op: 21 | test.nop 22 | 23 | {% endif %} 24 | -------------------------------------------------------------------------------- /ceph_salt/params_helper.py: -------------------------------------------------------------------------------- 1 | class Validator(): 2 | @classmethod 3 | def validate(cls, value): 4 | raise NotImplementedError() 5 | 6 | 7 | class Transformer(): 8 | @classmethod 9 | def transform(cls, value): 10 | raise NotImplementedError() 11 | 12 | 13 | class BooleanStringValidator(Validator): 14 | """Validate a string is in boolean type.""" 15 | @classmethod 16 | def validate(cls, value): 17 | return value.lower() in ['true', 'false', '1', '0'] 18 | 19 | 20 | class BooleanStringTransformer(Transformer): 21 | """Transform a boolean string to boolean type.""" 22 | @classmethod 23 | def transform(cls, value): 24 | return value.lower() in ['true', '1'] 25 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/cephtools.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {% if 'cephadm' in grains['ceph-salt']['roles'] %} 4 | 5 | {{ macros.begin_stage('Configure cephadm') }} 6 | 7 | {{ macros.begin_step('Install ceph packages') }} 8 | 9 | install cephadm: 10 | pkg.installed: 11 | - pkgs: 12 | - ceph-base 13 | - failhard: True 14 | 15 | {{ macros.end_step('Install ceph packages') }} 16 | 17 | {{ macros.begin_step('Run "cephadm check-host"') }} 18 | 19 | have cephadm check the host: 20 | cmd.run: 21 | - name: | 22 | cephadm check-host 23 | - failhard: True 24 | 25 | {{ macros.end_step('Run "cephadm check-host"') }} 26 | 27 | {{ macros.end_stage('Configure cephadm') }} 28 | 29 | {% endif %} 30 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/apparmor.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {{ macros.begin_stage('Install and configure AppArmor') }} 4 | 5 | aa-enabled: 6 | cmd.run: 7 | - onfail: 8 | - test: apparmor 9 | - failhard: True 10 | 11 | aa-teardown: 12 | cmd.run: 13 | - onlyif: 14 | - sh -c "type aa-teardown" 15 | - onfail: 16 | - test: apparmor 17 | - failhard: True 18 | 19 | stop apparmor: 20 | service.dead: 21 | - enable: False 22 | - failhard: True 23 | 24 | uninstall apparmor: 25 | pkg.removed: 26 | - pkgs: 27 | - apparmor 28 | - apparmor-utils 29 | - failhard: True 30 | 31 | apparmor: 32 | test.nop 33 | 34 | {{ macros.end_stage('Install and configure AppArmor') }} 35 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/files/chrony.conf.j2: -------------------------------------------------------------------------------- 1 | # {% include "ceph-salt/apply/files/managed-header.txt.j2" ignore missing %} 2 | {%- set time_servers = pillar['ceph-salt']['time_server']['server_hosts'] %} 3 | {%- if grains['id'] in time_servers %} 4 | {%- for server in pillar['ceph-salt']['time_server'].get('external_time_servers', []) %} 5 | pool {{ server }} iburst 6 | {%- endfor %} 7 | 8 | allow {{ pillar['ceph-salt']['time_server']['subnet'] }} 9 | {%- else %} {# grains['id'] in (time_servers) #} 10 | {%- for time_server in time_servers %} 11 | server {{ time_server }} iburst 12 | {%- endfor %} 13 | {%- endif %} {# grains['id'] in (time_servers) #} 14 | 15 | driftfile /var/lib/chrony/drift 16 | makestep 0.1 3 17 | rtcsync 18 | logdir /var/log/chrony 19 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/macros.yml: -------------------------------------------------------------------------------- 1 | {% macro send_event(action, type, desc) -%} 2 | {{ action }}_{{ type }}_{{ desc }}_enter: 3 | # module.run: 4 | # - event.send: 5 | # - tag: ceph-salt/{{ type }}/{{ action }} 6 | # - data: 7 | # desc: {{ desc }} 8 | ceph_salt.{{ action }}_{{ type }}: 9 | - name: {{ desc }} 10 | {%- endmacro %} 11 | 12 | {% macro begin_stage(desc) -%} 13 | {{ send_event('begin', 'stage', desc) }} 14 | {%- endmacro %} 15 | 16 | {% macro end_stage(desc) -%} 17 | {{ send_event('end', 'stage', desc) }} 18 | {%- endmacro %} 19 | 20 | {% macro begin_step(desc) -%} 21 | {{ send_event('begin', 'step', desc) }} 22 | {%- endmacro %} 23 | 24 | {% macro end_step(desc) -%} 25 | {{ send_event('end', 'step', desc) }} 26 | {%- endmacro %} 27 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/files/throughput.conf.j2: -------------------------------------------------------------------------------- 1 | # {% include "ceph-salt/apply/files/managed-header.txt.j2" ignore missing %} 2 | [main] 3 | include=throughput-performance 4 | dynamic_tuning = 0 5 | 6 | [disk] 7 | # hdparm -S 0 /dev/ 8 | # Sets the timeout to 0 - A value of zero means "timeouts are disabled": the 9 | # device will not automatically enter standby mode. 10 | spindown = 0 11 | # hdparm -B 254 /dev/ 12 | # Get/set Advanced Power Management feature. The highest I/O performance with a 13 | # setting of 254. 14 | apm = 254 15 | # Disable dynamic tuning for disk settings 16 | dynamic = 0 17 | 18 | [scsi_host] 19 | # ALPM is disabled; the link does not enter any low-power state when there is 20 | # no I/O on the disk. 21 | alpm = max_performance 22 | -------------------------------------------------------------------------------- /.github/workflows/linting.yml: -------------------------------------------------------------------------------- 1 | name: Linting 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Set up Python 3.7 13 | uses: actions/setup-python@v1 14 | with: 15 | python-version: 3.7 16 | - name: Install dependencies 17 | run: | 18 | python -m pip install --upgrade pip 19 | pip install -r requirements.txt 20 | - name: Lint with pylint 21 | run: | 22 | pylint ceph_salt 23 | - name: Lint with pycodestyle 24 | run: | 25 | pycodestyle ceph_salt 26 | - name: Lint tests with pylint 27 | run: | 28 | pylint tests 29 | - name: Lint tests with pycodestyle 30 | run: | 31 | pycodestyle tests 32 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/files/registries.conf.j2: -------------------------------------------------------------------------------- 1 | {% set registries = pillar['ceph-salt'].get('container', {}).get('registries', []) -%} 2 | # {% include "ceph-salt/apply/files/managed-header.txt.j2" ignore missing %} 3 | # For more information on this configuration file, see containers-registries.conf(5) 4 | 5 | # An array of host[:port] registries to try when pulling an unqualified image, in order 6 | unqualified-search-registries = ["docker.io"] 7 | 8 | {% for reg in registries %} 9 | [[registry]] 10 | {% if reg.prefix is defined %} 11 | prefix = "{{ reg.prefix }}" 12 | {% endif %} 13 | location = "{{ reg.location }}" 14 | {% if reg.insecure is defined %} 15 | insecure = {{ '{}'.format(reg.insecure) | lower }} 16 | {% endif %} 17 | {% if reg.blocked is defined %} 18 | blocked = {{ '{}'.format(reg.blocked) | lower }} 19 | {% endif %} 20 | {% endfor %} 21 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/tuned-latency.sls: -------------------------------------------------------------------------------- 1 | 2 | {% import 'macros.yml' as macros %} 3 | 4 | {% if 'latency' in grains['ceph-salt']['roles'] %} 5 | 6 | {{ macros.begin_stage('Configure tuned latency') }} 7 | 8 | install tuned: 9 | pkg.installed: 10 | - pkgs: 11 | - tuned 12 | - failhard: True 13 | 14 | /etc/tuned/ceph-latency/tuned.conf: 15 | file.managed: 16 | - source: salt://ceph-salt/apply/files/latency.conf.j2 17 | - template: jinja 18 | - makedirs: True 19 | - user: root 20 | - group: root 21 | - mode: 644 22 | - failhard: True 23 | 24 | start tuned for latency profile: 25 | service.running: 26 | - name: tuned 27 | - enable: True 28 | - failhard: True 29 | 30 | apply latency profile: 31 | cmd.run: 32 | - name: 'tuned-adm profile ceph-latency' 33 | - failhard: True 34 | 35 | {{ macros.end_stage('Configure tuned latency') }} 36 | 37 | {% endif %} 38 | 39 | tuned latency: 40 | test.nop 41 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/tuned-throughput.sls: -------------------------------------------------------------------------------- 1 | 2 | {% import 'macros.yml' as macros %} 3 | 4 | {% if 'throughput' in grains['ceph-salt']['roles'] %} 5 | 6 | {{ macros.begin_stage('Configure tuned throughput') }} 7 | 8 | install tuned: 9 | pkg.installed: 10 | - pkgs: 11 | - tuned 12 | - failhard: True 13 | 14 | /etc/tuned/ceph-throughput/tuned.conf: 15 | file.managed: 16 | - source: salt://ceph-salt/apply/files/throughput.conf.j2 17 | - template: jinja 18 | - makedirs: True 19 | - user: root 20 | - group: root 21 | - mode: 644 22 | - failhard: True 23 | 24 | start tuned for throughput profile: 25 | service.running: 26 | - name: tuned 27 | - enable: True 28 | - failhard: True 29 | 30 | apply throughput profile: 31 | cmd.run: 32 | - name: 'tuned-adm profile ceph-throughput' 33 | - failhard: True 34 | 35 | {{ macros.end_stage('Configure tuned throughput') }} 36 | 37 | {% endif %} 38 | 39 | tuned throughput: 40 | test.nop 41 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/ceph-admin.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {% if 'admin' in grains['ceph-salt']['roles'] %} 4 | 5 | {{ macros.begin_stage('Ensure cephadm MGR module is configured') }} 6 | 7 | {% set auth = pillar['ceph-salt'].get('container', {}).get('auth', {}) %} 8 | 9 | configure cephadm mgr module: 10 | cmd.run: 11 | - name: | 12 | ceph cephadm set-priv-key -i {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa 13 | ceph cephadm set-pub-key -i {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa.pub 14 | ceph cephadm set-user cephadm 15 | {%- if auth %} 16 | ceph cephadm registry-login -i /tmp/ceph-salt-registry-json 17 | {%- endif %} 18 | ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf true 19 | ceph config set mgr mgr/cephadm/use_repo_digest true --force 20 | ceph config set mgr mgr/cephadm/container_init true 21 | - failhard: True 22 | 23 | {{ macros.end_stage('Ensure cephadm MGR module is configured') }} 24 | 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/tuned-off.sls: -------------------------------------------------------------------------------- 1 | 2 | {% import 'macros.yml' as macros %} 3 | 4 | {% if 'latency' not in grains['ceph-salt']['roles'] and 'throughput' not in grains['ceph-salt']['roles'] %} 5 | 6 | {{ macros.begin_stage('Disable tuned') }} 7 | 8 | stop tuned: 9 | service.dead: 10 | - name: tuned 11 | - enable: False 12 | - failhard: True 13 | 14 | remove tuned ceph latency: 15 | file.absent: 16 | - name: /etc/tuned/ceph-latency/ 17 | - failhard: True 18 | 19 | remove tuned ceph throughput: 20 | file.absent: 21 | - name: /etc/tuned/ceph-throughput/ 22 | - failhard: True 23 | 24 | remove tuned ceph mon: 25 | file.absent: 26 | - name: /etc/tuned/ceph-mon/ 27 | - failhard: True 28 | 29 | remove tuned ceph mgr: 30 | file.absent: 31 | - name: /etc/tuned/ceph-mgr/ 32 | - failhard: True 33 | 34 | remove tuned ceph osd: 35 | file.absent: 36 | - name: /etc/tuned/ceph-osd/ 37 | - failhard: True 38 | 39 | {{ macros.end_stage('Disable tuned') }} 40 | 41 | {% endif %} 42 | 43 | tuned off: 44 | test.nop 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 SUSE LLC. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/common/install-cephadm.sls: -------------------------------------------------------------------------------- 1 | # This little hack ensures that the cephadm package is installed at jinja 2 | # *compile* time on all salt minions. The cephadm package itself ensures 3 | # the cephadm user and group are created, so we'll be able to rely on those 4 | # things existing later when the states are applied to pick up the correct 5 | # home directory, gid, etc. The fact that the next line is a comment 6 | # doesn't matter, it'll still be expanded at jinja compile time and the 7 | # package will be installed correctly (either that or it'll fail with an 8 | # appropriate error message if the package can't be installed for some 9 | # reason). 10 | # 11 | # {{ salt['pkg.install']('cephadm') }} 12 | # 13 | # One irritation is that ideally, cephadm would only be installed on nodes 14 | # with the cephadm role. Unfortunately, ceph-salt uses the cephadm user's 15 | # ssh keys to ssh between nodes to do things like check if grains are set 16 | # on remote hosts (e.g. when setting up time sync). This means we need the 17 | # cephadm package installed everywhere to ensure the user and home directory 18 | # are present. 19 | -------------------------------------------------------------------------------- /Jenkinsfile.integration: -------------------------------------------------------------------------------- 1 | #!groovy 2 | 3 | // base jenkins slave 4 | def jenkins_slave_base='storage-compute' 5 | 6 | def get_ceph_salt_git_branch(change_id) { 7 | if (!change_id || change_id.isEmpty()) { 8 | return "master" 9 | } else { 10 | return "origin/pr-merged/" + String.valueOf(change_id) 11 | } 12 | } 13 | 14 | pipeline { 15 | agent none 16 | 17 | options { parallelsAlwaysFailFast() } 18 | 19 | stages 20 | { 21 | stage ('Invoke sesdev pipeline') { 22 | agent { 23 | label "${jenkins_slave_base}" 24 | } 25 | steps { 26 | build job: 'sesdev-integration/master', parameters: [ 27 | // Note: the origin/pr-merged/XXX syntax is possible due to https://github.com/SUSE/sesdev/pull/198 28 | string(name: 'CEPH_SALT_GIT_BRANCH', value: get_ceph_salt_git_branch(env.CHANGE_ID)), 29 | string(name: 'CUSTOM_JOB_NAME', value: "ceph-salt PR-" + String.valueOf(env.CHANGE_ID)), 30 | string(name: 'CUSTOM_JOB_DESC', value: String.valueOf(env.CHANGE_URL)) 31 | ] 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /ceph_salt/logging_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | class LoggingUtil: 5 | log_file = None 6 | 7 | @classmethod 8 | def setup_logging(cls, log_level, log_file): 9 | """ 10 | Logging configuration 11 | """ 12 | cls.log_file = log_file 13 | if log_level == "silent": 14 | return 15 | 16 | logging.config.dictConfig({ 17 | 'version': 1, 18 | 'disable_existing_loggers': False, 19 | 'formatters': { 20 | 'standard': { 21 | 'format': '%(asctime)s [%(levelname)s] [%(name)s] %(message)s' 22 | }, 23 | }, 24 | 'handlers': { 25 | 'file': { 26 | 'level': log_level.upper(), 27 | 'filename': log_file, 28 | 'class': 'logging.FileHandler', 29 | 'formatter': 'standard' 30 | }, 31 | }, 32 | 'loggers': { 33 | '': { 34 | 'handlers': ['file'], 35 | 'level': log_level.upper(), 36 | 'propagate': True, 37 | } 38 | } 39 | }) 40 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/_modules/ceph_orch.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import json 3 | 4 | def configured(): 5 | if not __salt__['file.file_exists']("/etc/ceph/ceph.conf"): 6 | return False 7 | if not __salt__['file.file_exists']("/etc/ceph/ceph.client.admin.keyring"): 8 | return False 9 | status_ret = __salt__['cmd.run_all']("timeout 60 ceph orch status --format=json") 10 | if status_ret['retcode'] != 0: 11 | return False 12 | status = json.loads(status_ret['stdout']) 13 | return status.get('available', False) 14 | 15 | def ceph_configured(): 16 | if not __salt__['file.file_exists']("/etc/ceph/ceph.conf"): 17 | return False 18 | if not __salt__['file.file_exists']("/etc/ceph/ceph.client.admin.keyring"): 19 | return False 20 | status_ret = __salt__['cmd.run_all']("ceph -s") 21 | return status_ret['retcode'] == 0 22 | 23 | def host_ls(): 24 | ret = __salt__['cmd.run']("ceph orch host ls --format=json") 25 | return json.loads(ret) 26 | 27 | def fsid(): 28 | ret = __salt__['cmd.run_all']("ceph -s --format=json") 29 | if ret['retcode'] == 0: 30 | status = json.loads(ret['stdout']) 31 | return status.get('fsid', None) 32 | return None 33 | -------------------------------------------------------------------------------- /ceph_salt/exceptions.py: -------------------------------------------------------------------------------- 1 | class CephSaltException(Exception): 2 | pass 3 | 4 | 5 | class CephNodeHasRolesException(CephSaltException): 6 | def __init__(self, minion_id, roles): 7 | super(CephNodeHasRolesException, self).__init__( 8 | "Cannot remove host '{}' because it has roles defined: {}".format(minion_id, roles)) 9 | 10 | 11 | class SaltCallException(CephSaltException): 12 | def __init__(self, target, func, ret): 13 | super(SaltCallException, self).__init__( 14 | "Salt call target='{}' func='{}' failed: {}".format(target, func, ret)) 15 | 16 | 17 | class ValidationException(CephSaltException): 18 | pass 19 | 20 | 21 | class PillarFileNotPureYaml(CephSaltException): 22 | def __init__(self, top_file_path): 23 | super(PillarFileNotPureYaml, self).__init__( 24 | "Salt pillar file '{}' may contain Jinja2 expressions".format(top_file_path)) 25 | 26 | 27 | class MinionDoesNotExistInConfiguration(CephSaltException): 28 | def __init__(self, minion_id): 29 | super(MinionDoesNotExistInConfiguration, self).__init__( 30 | "Minion '{}' does not exist in configuration".format(minion_id)) 31 | 32 | 33 | class ParamsException(CephSaltException): 34 | pass 35 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/reboot/reboot.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | install required packages: 4 | pkg.installed: 5 | - pkgs: 6 | - lsof 7 | - failhard: True 8 | 9 | {{ macros.begin_stage('Check if reboot is needed') }} 10 | check if reboot is needed: 11 | ceph_salt.set_reboot_needed: 12 | - force: {{ pillar['ceph-salt'].get('force-reboot', False) }} 13 | - failhard: True 14 | {{ macros.end_stage('Check if reboot is needed') }} 15 | 16 | # if ceph cluster already exists, then minions are rebooted sequentially (orchestrated reboot) 17 | # otherwise minions are rebooted in parallel 18 | {% if pillar['ceph-salt'].get('execution', {}).get('deployed') != False %} 19 | 20 | wait for admin host: 21 | ceph_orch.set_admin_host: 22 | - if_grain: ceph-salt:execution:reboot_needed 23 | - failhard: True 24 | 25 | wait for ancestor minion: 26 | ceph_salt.wait_for_ancestor_minion_grain: 27 | - grain: ceph-salt:execution:rebooted 28 | - if_grain: ceph-salt:execution:reboot_needed 29 | - failhard: True 30 | 31 | wait for ceph orch ok-to-stop: 32 | ceph_orch.wait_for_ceph_orch_host_ok_to_stop: 33 | - if_grain: ceph-salt:execution:reboot_needed 34 | - failhard: True 35 | 36 | {% endif %} 37 | 38 | reboot: 39 | ceph_salt.reboot_if_needed: 40 | - failhard: True 41 | -------------------------------------------------------------------------------- /.github/workflows/tox.yml: -------------------------------------------------------------------------------- 1 | name: "tox" 2 | 3 | on: 4 | - pull_request 5 | 6 | jobs: 7 | pytest: 8 | runs-on: ubuntu-20.04 9 | strategy: 10 | matrix: 11 | include: 12 | - tox-env: py36 13 | py-ver: 3.6 14 | - tox-env: py38 15 | py-ver: 3.8 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python ${{ matrix.py-ver }} 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: ${{ matrix.py-ver }} 23 | 24 | - name: Install dependencies 25 | run: | 26 | python --version 27 | python -m pip install --upgrade pip 28 | pip install tox tox-gh-actions 29 | - name: Test with tox 30 | env: 31 | TOXENV: ${{ matrix.tox-env }} 32 | run: tox 33 | - name: Upload Coverage to Codecov 34 | uses: codecov/codecov-action@v1 35 | lint: 36 | runs-on: ubuntu-latest 37 | steps: 38 | - uses: actions/checkout@v2 39 | - name: Set up Python 3.8 40 | uses: actions/setup-python@v2 41 | with: 42 | python-version: 3.8 43 | - name: Install dependencies 44 | run: | 45 | python -m pip install --upgrade pip 46 | pip install tox tox-gh-actions 47 | - name: Run lint with tox 48 | env: 49 | TOXENV: lint 50 | run: tox 51 | 52 | -------------------------------------------------------------------------------- /ceph_salt/validate/salt_minion.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | 3 | from ..exceptions import ValidationException 4 | from ..salt_utils import SaltClient 5 | 6 | 7 | class UnableToSyncAll(ValidationException): 8 | def __init__(self): 9 | super(UnableToSyncAll, self).__init__( 10 | "Sync failed, please run: " 11 | "\"salt -G 'ceph-salt:member' saltutil.sync_all\" manually and fix " 12 | "the problems reported") 13 | 14 | 15 | class UnableToSyncModules(ValidationException): 16 | def __init__(self, target): 17 | super(UnableToSyncModules, self).__init__( 18 | "Sync failed, please run: " 19 | "\"salt -G '{}' saltutil.sync_modules\" manually and fix " 20 | "the problems reported".format(target)) 21 | 22 | 23 | def sync_all(): 24 | with contextlib.redirect_stdout(None): 25 | result = SaltClient.local_cmd('ceph-salt:member', 'saltutil.sync_all', tgt_type='grain') 26 | for minion, value in result.items(): 27 | if not value: 28 | raise UnableToSyncAll() 29 | 30 | 31 | def sync_modules(target='ceph-salt:member'): 32 | with contextlib.redirect_stdout(None): 33 | result = SaltClient.local_cmd(target, 'saltutil.sync_modules', tgt_type='grain') 34 | for value in result: 35 | if not value: 36 | raise UnableToSyncModules(target) 37 | -------------------------------------------------------------------------------- /HOWTORELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Procedure 2 | 3 | These are the steps to make a release for version ``: 4 | 5 | 1. Make sure you are working on the current tip of the master branch. 6 | 2. Make sure the merged PRs of all important changes have the "Add To Changelog" label: 7 | https://github.com/ceph/ceph-salt/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Amerged+label%3A%22Add+To+Changelog%22+ 8 | 3. Update `CHANGELOG.md` with all important changes introduced since previous version. 9 | - Create a new section `[] ` and move all entries 10 | from the `[Unreleased]` section to the new section. 11 | - Make sure all github issues resolved in this release are referenced in the changelog. 12 | - Update the links at the bottom of the file. 13 | 4. Update version number in `ceph-salt.spec` to `Version: `. 14 | 5. Create a commit with title `Bump to v` containing the 15 | modifications to `CHANGELOG.md` made in the previous two steps. 16 | 6. create and merge a new PR with this bump. 17 | 7. go to https://github.com/ceph/ceph-salt/releases and create a new release 18 | 8. Remove the "Add To Changelog" labels from all the merged PRs 19 | 9. Verify that no merged PRs have "Add To Changelog" label: 20 | https://github.com/ceph/ceph-salt/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Amerged+label%3A%22Add+To+Changelog%22+ 21 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/time-prep.sls: -------------------------------------------------------------------------------- 1 | {% if pillar['ceph-salt']['time_server']['enabled'] %} 2 | 3 | {% import 'macros.yml' as macros %} 4 | 5 | {{ macros.begin_stage('Prepare cluster for time synchronization') }} 6 | 7 | {{ macros.begin_step('Install chrony package') }} 8 | install chrony: 9 | pkg.installed: 10 | - pkgs: 11 | - chrony 12 | - refresh: True 13 | - failhard: True 14 | {{ macros.end_step('Install chrony package') }} 15 | 16 | service_reload: 17 | module.run: 18 | - name: service.systemctl_reload 19 | - failhard: True 20 | 21 | {{ macros.begin_step('Configure chrony service') }} 22 | /etc/chrony.conf: 23 | file.managed: 24 | - source: 25 | - salt://ceph-salt/apply/files/chrony.conf.j2 26 | - template: jinja 27 | - user: root 28 | - group: root 29 | - mode: '0644' 30 | - makedirs: True 31 | - backup: minion 32 | - failhard: True 33 | 34 | {{ macros.end_step('Configure chrony service') }} 35 | 36 | {{ macros.begin_step('Restart chrony service') }} 37 | stop chronyd: 38 | service.dead: 39 | - name: chronyd 40 | - failhard: True 41 | 42 | start chronyd: 43 | service.running: 44 | - name: chronyd 45 | - enable: True 46 | - failhard: True 47 | {{ macros.end_step('Restart chrony service') }} 48 | 49 | {{ macros.end_stage('Prepare cluster for time synchronization') }} 50 | 51 | {% endif %} 52 | 53 | prevent empty time-prep: 54 | test.nop 55 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/container.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {{ macros.begin_stage('Set up container environment') }} 4 | 5 | {% if pillar['ceph-salt']['container']['registries_enabled'] %} 6 | 7 | {{ macros.begin_step('Configure container image registries') }} 8 | 9 | /etc/containers/registries.conf: 10 | file.managed: 11 | - source: 12 | - salt://ceph-salt/apply/files/registries.conf.j2 13 | - template: jinja 14 | - user: root 15 | - group: root 16 | - mode: '0644' 17 | - backup: minion 18 | - failhard: True 19 | 20 | {{ macros.end_step('Configure container image registries') }} 21 | 22 | {% endif %} 23 | 24 | {% if grains['id'] == pillar['ceph-salt'].get('bootstrap_minion') or 'admin' in grains['ceph-salt']['roles'] %} 25 | {% set auth = pillar['ceph-salt'].get('container', {}).get('auth', {}) %} 26 | {% if auth %} 27 | {{ macros.begin_step('Set container registry credentials') }} 28 | 29 | create ceph-salt-registry-json: 30 | file.managed: 31 | - name: /tmp/ceph-salt-registry-json 32 | - source: 33 | - salt://ceph-salt/files/registry-login-json.j2 34 | - template: jinja 35 | - user: root 36 | - group: root 37 | - mode: '0600' 38 | - backup: minion 39 | - failhard: True 40 | 41 | {{ macros.end_step('Set container registry credentials') }} 42 | {% endif %} 43 | {% endif %} 44 | 45 | 46 | {{ macros.end_stage('Set up container environment') }} 47 | -------------------------------------------------------------------------------- /tests/test_grains_manager.py: -------------------------------------------------------------------------------- 1 | from ceph_salt.salt_utils import GrainsManager 2 | from . import SaltMockTestCase 3 | 4 | 5 | class GrainsManagerTest(SaltMockTestCase): 6 | 7 | def setUp(self): 8 | super(GrainsManagerTest, self).setUp() 9 | GrainsManager.set_grain('test', 'key', 'value') 10 | 11 | def test_grains_set(self): 12 | self.assertGrains('test', 'key', 'value') 13 | 14 | def test_grains_get(self): 15 | value = GrainsManager.get_grain('test', 'key') 16 | self.assertDictEqual(value, {'test': 'value'}) 17 | 18 | def test_grains_del(self): 19 | GrainsManager.del_grain('test', 'key') 20 | self.assertNotInGrains('test', 'key') 21 | 22 | def test_grains_filter_by(self): 23 | GrainsManager.set_grain('node1', 'ceph-salt', {'member': True, 24 | 'roles': ['mon'], 25 | 'execution': {}}) 26 | GrainsManager.set_grain('node2', 'ceph-salt', {'member': True, 27 | 'roles': ['mgr'], 28 | 'execution': {}}) 29 | GrainsManager.set_grain('node3', 'ceph-salt', {'member': True, 30 | 'roles': ['storage'], 31 | 'execution': {}}) 32 | result = GrainsManager.filter_by('ceph-salt:member') 33 | self.assertEqual(set(result), {'node1', 'node2', 'node3'}) 34 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = ceph-salt 3 | summary = CLI tool to bootstrap Ceph clusters 4 | description-file = 5 | README.md 6 | CHANGELOG.md 7 | home-page = https://github.com/ceph/ceph-salt 8 | requires-dist = setuptools 9 | license = MIT License 10 | license_file = LICENSE 11 | classifier = 12 | Development Status :: 5 - Production/Stable 13 | Environment :: Console 14 | License :: OSI Approved :: MIT License 15 | Natural Language :: English 16 | Operating System :: POSIX 17 | Programming Language :: Python :: 3 18 | Topic :: System :: Clustering 19 | Topic :: System :: Distributed Computing 20 | Topic :: Utilities 21 | 22 | [options] 23 | install_requires = 24 | Click >= 6.7 25 | configshell-fb >= 1.1 26 | pycryptodomex >= 3.4.6 27 | PyYAML >= 5.1.2 28 | salt >= 3000 29 | 30 | packages = 31 | ceph_salt 32 | ceph_salt.validate 33 | 34 | tests_require = 35 | pytest 36 | 37 | setup_requries = 38 | pytest-runner 39 | 40 | [aliases] 41 | test=pytest 42 | 43 | [options.entry_points] 44 | console_scripts = 45 | ceph-salt = ceph_salt:ceph_salt_main 46 | 47 | [options.extras_require] 48 | dev = 49 | mock==3.0.5 50 | pycodestyle==2.5.0 51 | pyfakefs==3.7 52 | pylint==2.4.4 53 | pytest==5.3.1 54 | pytest-cov==2.8.1 55 | pytest-runner==5.2 56 | 57 | [pycodestyle] 58 | max-line-length = 100 59 | ignore = 60 | W605 61 | 62 | [tool:pytest] 63 | testpaths = tests 64 | 65 | [coverage:paths] 66 | source = 67 | ceph_salt 68 | 69 | [coverage:run] 70 | omit = 71 | tests/* 72 | */python*/* 73 | 74 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/update/update.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {{ macros.begin_stage('Update all packages') }} 4 | 5 | install required packages: 6 | pkg.installed: 7 | - pkgs: 8 | - lsof 9 | - failhard: True 10 | 11 | update packages: 12 | module.run: 13 | - name: pkg.upgrade 14 | - failhard: True 15 | 16 | {{ macros.end_stage('Update all packages') }} 17 | 18 | {% if pillar['ceph-salt'].get('execution', {}).get('reboot-if-needed', False) %} 19 | 20 | {{ macros.begin_stage('Check if reboot is needed') }} 21 | check if reboot is needed: 22 | ceph_salt.set_reboot_needed: 23 | - failhard: True 24 | {{ macros.end_stage('Check if reboot is needed') }} 25 | 26 | # if ceph cluster already exists, then minions are rebooted sequentially (orchestrated reboot) 27 | # otherwise minions are rebooted in parallel 28 | {% if pillar['ceph-salt'].get('execution', {}).get('deployed') != False %} 29 | 30 | wait for admin host: 31 | ceph_orch.set_admin_host: 32 | - if_grain: ceph-salt:execution:reboot_needed 33 | - failhard: True 34 | 35 | wait for ancestor minion: 36 | ceph_salt.wait_for_ancestor_minion_grain: 37 | - grain: ceph-salt:execution:updated 38 | - if_grain: ceph-salt:execution:reboot_needed 39 | - failhard: True 40 | 41 | wait for ceph orch ok-to-stop: 42 | ceph_orch.wait_for_ceph_orch_host_ok_to_stop: 43 | - if_grain: ceph-salt:execution:reboot_needed 44 | - failhard: True 45 | 46 | {% endif %} 47 | 48 | reboot: 49 | ceph_salt.reboot_if_needed: 50 | - failhard: True 51 | 52 | {% else %} 53 | 54 | skip reboot: 55 | test.nop 56 | 57 | {% endif %} 58 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/stop/stop.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | check fsid: 4 | ceph_salt.check_fsid: 5 | - formula: ceph-salt.stop 6 | - failhard: True 7 | 8 | # first admin minion is the one who will stop services 9 | {% set admin_minion = pillar['ceph-salt']['minions']['admin'][0] %} 10 | 11 | {% if grains['id'] == admin_minion %} 12 | 13 | {{ macros.begin_stage('Ensure noout OSD flag is set') }} 14 | set noout osd flag: 15 | ceph_orch.set_osd_flag: 16 | - flag: noout 17 | - failhard: True 18 | {{ macros.end_stage('Ensure noout OSD flag is set') }} 19 | 20 | {{ macros.begin_stage('Stop ceph services') }} 21 | 22 | {% for service in ['nfs', 'iscsi', 'rgw', 'mds', 'prometheus', 'grafana', 'node-exporter', 'alertmanager', 'rbd-mirror', 'crash', 'osd'] %} 23 | 24 | {{ macros.begin_step("Stop '" ~ service ~ "'") }} 25 | stop {{ service }}: 26 | ceph_orch.stop_service: 27 | - service: {{ service }} 28 | - failhard: True 29 | 30 | wait {{ service }} stopped: 31 | ceph_orch.wait_until_service_stopped: 32 | - service: {{ service }} 33 | - failhard: True 34 | {{ macros.end_step("Stop '" ~ service ~ "'") }} 35 | 36 | {% endfor %} 37 | 38 | {{ macros.end_stage('Stop ceph services') }} 39 | 40 | {% else %} 41 | 42 | {{ macros.begin_stage('Wait until ' ~ admin_minion ~ ' stops all ceph services') }} 43 | wait for admin stopped: 44 | ceph_salt.wait_for_grain: 45 | - grain: ceph-salt:execution:stopped 46 | - hosts: [ {{ admin_minion }} ] 47 | - failhard: True 48 | {{ macros.end_stage('Wait until ' ~ admin_minion ~ ' stops all ceph services') }} 49 | 50 | {% endif %} 51 | 52 | # Stop remaining services (mgr, mon) 53 | stop cluster: 54 | ceph_orch.stop_ceph_fsid: 55 | - failhard: True 56 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/cephconfigure.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {% if 'cephadm' in grains['ceph-salt']['roles'] or 'admin' in grains['ceph-salt']['roles'] %} 4 | 5 | find an admin host: 6 | ceph_orch.set_admin_host: 7 | - failhard: True 8 | 9 | {% endif %} 10 | 11 | 12 | {% if 'admin' in grains['ceph-salt']['roles'] %} 13 | 14 | {{ macros.begin_stage('Ensure ceph.conf and keyring are present') }} 15 | 16 | copy ceph.conf and keyring from admin node: 17 | ceph_orch.copy_ceph_conf_and_keyring_from_admin: 18 | - failhard: True 19 | 20 | {{ macros.end_stage('Ensure ceph.conf and keyring are present') }} 21 | 22 | {% set admin_minion = pillar['ceph-salt'].get('bootstrap_minion', pillar['ceph-salt']['minions']['admin'][0]) %} 23 | 24 | {% if grains['id'] == admin_minion %} 25 | 26 | {{ macros.begin_stage('Ensure cephadm MGR module is enabled') }} 27 | 28 | enable cephadm mgr module: 29 | cmd.run: 30 | - name: | 31 | ceph config-key set mgr/cephadm/ssh_identity_key -i {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa 32 | ceph config-key set mgr/cephadm/ssh_identity_pub -i {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa.pub 33 | ceph config-key set mgr/cephadm/ssh_user cephadm 34 | ceph mgr module enable cephadm && \ 35 | ceph orch set backend cephadm 36 | - failhard: True 37 | 38 | {{ macros.end_stage('Ensure cephadm MGR module is enabled') }} 39 | 40 | {% endif %} 41 | 42 | {% endif %} 43 | 44 | {% if 'cephadm' in grains['ceph-salt']['roles'] or 'admin' in grains['ceph-salt']['roles'] %} 45 | 46 | {{ macros.begin_stage('Wait until ceph orch is available') }} 47 | 48 | wait for ceph orch available: 49 | ceph_orch.wait_until_ceph_orch_available: 50 | - failhard: True 51 | 52 | {{ macros.end_stage('Wait until ceph orch is available') }} 53 | 54 | {% endif %} -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/time-sync.sls: -------------------------------------------------------------------------------- 1 | {% if pillar['ceph-salt']['time_server']['enabled'] %} 2 | {% set time_server = pillar['ceph-salt']['time_server']['server_hosts'][0] %} 3 | {% set time_server_is_minion = time_server in pillar['ceph-salt']['minions']['all'] %} 4 | 5 | {% import 'macros.yml' as macros %} 6 | 7 | {{ macros.begin_stage('Sync clocks') }} 8 | 9 | /tmp/chrony_sync_clock.sh: 10 | file.managed: 11 | - source: 12 | - salt://ceph-salt/apply/files/chrony_sync_clock.sh 13 | - user: root 14 | - group: root 15 | - mode: '0755' 16 | - makedirs: True 17 | - backup: minion 18 | - failhard: True 19 | 20 | {% if time_server_is_minion and grains['id'] != time_server %} 21 | 22 | {{ macros.begin_step('Wait for time server node to signal that it has synced its clock') }} 23 | wait for time server sync: 24 | ceph_salt.wait_for_grain: 25 | - grain: ceph-salt:execution:timeserversynced 26 | - hosts: [ {{ time_server }} ] 27 | - failhard: True 28 | {{ macros.end_step('Wait for time server node to signal that it has synced its clock') }} 29 | 30 | {% endif %} 31 | 32 | {{ macros.begin_step('Force clock sync now') }} 33 | 34 | run sync script: 35 | cmd.run: 36 | - name: | 37 | /tmp/chrony_sync_clock.sh 38 | - failhard: True 39 | 40 | {{ macros.end_step('Force clock sync now') }} 41 | 42 | {% if grains['id'] == time_server %} 43 | {{ macros.begin_step('Signal that time server node has synced its clock') }} 44 | set timeserversynced: 45 | grains.present: 46 | - name: ceph-salt:execution:timeserversynced 47 | - value: True 48 | {{ macros.end_step('Signal that time server node has synced its clock') }} 49 | {% endif %} 50 | 51 | delete clock sync script: 52 | file.absent: 53 | - name: | 54 | /tmp/chrony_sync_clock.sh 55 | - failhard: True 56 | 57 | {{ macros.end_stage('Sync clocks') }} 58 | 59 | {% endif %} 60 | 61 | prevent empty time-sync: 62 | test.nop 63 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/common/sshkey.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {{ macros.begin_stage('Ensure SSH keys are configured') }} 4 | 5 | install sudo: 6 | pkg.installed: 7 | - pkgs: 8 | - sudo 9 | - failhard: True 10 | 11 | configure sudoers: 12 | file.append: 13 | - name: /etc/sudoers.d/ceph-salt 14 | - text: 15 | - "cephadm ALL=NOPASSWD: /usr/bin/ceph -s" 16 | - "cephadm ALL=NOPASSWD: /usr/bin/ceph orch host add *" 17 | - "cephadm ALL=NOPASSWD: /usr/bin/ceph orch host label *" 18 | - "cephadm ALL=NOPASSWD: /usr/bin/ceph orch host ok-to-stop *" 19 | - "cephadm ALL=NOPASSWD: /usr/bin/ceph orch status --format=json" 20 | - "cephadm ALL=NOPASSWD: /usr/bin/python3" 21 | - "cephadm ALL=NOPASSWD: /usr/bin/rsync" 22 | 23 | # make sure .ssh is present with the right permissions 24 | create ssh dir: 25 | file.directory: 26 | - name: {{ salt['user.info']('cephadm').home }}/.ssh 27 | - user: cephadm 28 | - group: {{ salt['user.info']('cephadm').gid }} 29 | - mode: '0700' 30 | - makedirs: True 31 | - failhard: True 32 | 33 | # private key 34 | create ceph-salt-ssh-id_rsa: 35 | file.managed: 36 | - name: {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa 37 | - user: cephadm 38 | - group: {{ salt['user.info']('cephadm').gid }} 39 | - mode: '0600' 40 | - contents_pillar: ceph-salt:ssh:private_key 41 | - failhard: True 42 | 43 | # public key 44 | create ceph-salt-ssh-id_rsa.pub: 45 | file.managed: 46 | - name: {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa.pub 47 | - user: cephadm 48 | - group: {{ salt['user.info']('cephadm').gid }} 49 | - mode: '0644' 50 | - contents_pillar: ceph-salt:ssh:public_key 51 | - failhard: True 52 | 53 | # add public key to authorized_keys 54 | install ssh key: 55 | ssh_auth.present: 56 | - user: cephadm 57 | - comment: ssh_key_created_by_ceph_salt 58 | - config: /%h/.ssh/authorized_keys 59 | - name: {{ pillar['ceph-salt']['ssh']['public_key'] }} 60 | - failhard: True 61 | 62 | {{ macros.end_stage('Ensure SSH keys are configured') }} 63 | -------------------------------------------------------------------------------- /ceph_salt/validate/salt_master.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import shutil 3 | import subprocess 4 | 5 | from ..exceptions import ValidationException 6 | from ..salt_utils import SaltClient, PillarManager 7 | 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class NoSaltMasterProcess(ValidationException): 13 | def __init__(self): 14 | super(NoSaltMasterProcess, self).__init__('No salt-master process is running') 15 | 16 | 17 | class SaltMasterNotInstalled(ValidationException): 18 | def __init__(self): 19 | super(SaltMasterNotInstalled, self).__init__('salt-master is not installed') 20 | 21 | 22 | class NoPillarDirectoryConfigured(ValidationException): 23 | def __init__(self): 24 | super(NoPillarDirectoryConfigured, self).__init__( 25 | "Salt master 'pillar_roots' configuration does not have any directory") 26 | 27 | 28 | class CephSaltPillarNotConfigured(ValidationException): 29 | def __init__(self): 30 | super(CephSaltPillarNotConfigured, self).__init__("ceph-salt pillar not configured") 31 | 32 | 33 | def check_salt_master(): 34 | try: 35 | logger.info("checking if salt-master is installed") 36 | if shutil.which('salt-master') is None: 37 | logger.error('salt-master is not installed') 38 | raise SaltMasterNotInstalled() 39 | 40 | logger.info("checking if salt-master process is running") 41 | count = subprocess.check_output(['pgrep', '-c', 'salt-master']) 42 | if int(count) > 0: 43 | return 44 | except subprocess.CalledProcessError as ex: 45 | logger.exception(ex) 46 | logger.error("no salt-master process found") 47 | raise NoSaltMasterProcess() 48 | 49 | 50 | def check_ceph_salt_pillar(): 51 | logger.info("checking if pillar directory is configured") 52 | if not SaltClient.pillar_fs_path(): 53 | logger.info("salt-master pillar_roots configuration does not have any directory") 54 | raise NoPillarDirectoryConfigured() 55 | 56 | logger.info("checking if ceph-salt pillar is correctly configured") 57 | if not PillarManager.pillar_installed(): 58 | logger.error("ceph-salt is not present in the pillar") 59 | raise CephSaltPillarNotConfigured() 60 | 61 | 62 | def check_salt_master_status(): 63 | check_salt_master() 64 | check_ceph_salt_pillar() 65 | -------------------------------------------------------------------------------- /tests/test_validate_salt_master.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | from mock import patch 4 | 5 | from ceph_salt.validate.salt_master import check_salt_master, SaltMasterNotInstalled, \ 6 | NoSaltMasterProcess, check_ceph_salt_pillar, NoPillarDirectoryConfigured, \ 7 | CephSaltPillarNotConfigured 8 | 9 | from . import SaltMockTestCase 10 | 11 | 12 | # pylint: disable=unused-argument 13 | 14 | 15 | class ValidateSaltMasterTest(SaltMockTestCase): 16 | 17 | @patch('shutil.which', return_value=None) 18 | def test_salt_master_installed(self, *args): 19 | with self.assertRaises(SaltMasterNotInstalled) as ctx: 20 | check_salt_master() 21 | 22 | self.assertEqual(str(ctx.exception), "salt-master is not installed") 23 | 24 | @patch('shutil.which', return_value="/usr/bin/salt-master") 25 | @patch('subprocess.check_output', return_value="0") 26 | def test_salt_master_running(self, *args): 27 | with self.assertRaises(NoSaltMasterProcess) as ctx: 28 | check_salt_master() 29 | 30 | self.assertEqual(str(ctx.exception), "No salt-master process is running") 31 | 32 | @patch('shutil.which', return_value="/usr/bin/salt-master") 33 | @patch('subprocess.check_output', side_effect=subprocess.CalledProcessError(1, "pgrep")) 34 | def test_salt_master_running_exception(self, *args): 35 | with self.assertRaises(NoSaltMasterProcess) as ctx: 36 | check_salt_master() 37 | 38 | self.assertEqual(str(ctx.exception), "No salt-master process is running") 39 | 40 | @patch('shutil.which', return_value="/usr/bin/salt-master") 41 | @patch('subprocess.check_output', return_value="1") 42 | def test_salt_master_ok(self, *args): 43 | check_salt_master() 44 | 45 | def test_ceph_salt_pillar_directory(self): 46 | self.master_config.opts = {} 47 | with self.assertRaises(NoPillarDirectoryConfigured) as ctx: 48 | check_ceph_salt_pillar() 49 | 50 | self.assertEqual(str(ctx.exception), 51 | "Salt master 'pillar_roots' configuration does not have any directory") 52 | 53 | def test_ceph_salt_pillar_installed(self): 54 | self.fs.remove_object('/srv/pillar/ceph-salt.sls') 55 | with self.assertRaises(CephSaltPillarNotConfigured) as ctx: 56 | check_ceph_salt_pillar() 57 | 58 | self.assertEqual(str(ctx.exception), "ceph-salt pillar not configured") 59 | self.fs.create_file('/srv/pillar/ceph-salt.sls') 60 | -------------------------------------------------------------------------------- /ceph_salt/terminal_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from functools import wraps 4 | 5 | 6 | class PrettyPrinter: 7 | """ 8 | Helper class to pretty print 9 | """ 10 | 11 | _colors_enabled = True 12 | 13 | class Colors: 14 | """ 15 | Color enum 16 | """ 17 | RED = '\x1B[38;5;196m' 18 | GREEN = '\x1B[38;5;83m' 19 | ORANGE = '\x1B[38;5;214m' 20 | ENDC = '\x1B[0m' 21 | 22 | @classmethod 23 | def disable_colors(cls): 24 | cls._colors_enabled = False 25 | 26 | @classmethod 27 | def _format(cls, color, text): 28 | """ 29 | Generic pretty print string formatter 30 | """ 31 | if not cls._colors_enabled: 32 | color = '' 33 | return u"{}{}{}".format(color, text, cls.Colors.ENDC) 34 | 35 | @staticmethod 36 | def green(text): 37 | """ 38 | Formats text as green 39 | """ 40 | return PrettyPrinter._format(PrettyPrinter.Colors.GREEN, text) 41 | 42 | @staticmethod 43 | def red(text): 44 | """ 45 | Formats text as red 46 | """ 47 | return PrettyPrinter._format(PrettyPrinter.Colors.RED, text) 48 | 49 | @staticmethod 50 | def orange(text): 51 | """ 52 | Formats text as orange 53 | """ 54 | return PrettyPrinter._format(PrettyPrinter.Colors.ORANGE, text) 55 | 56 | @classmethod 57 | def println(cls, text=None): 58 | """ 59 | Prints text as is with newline in the end 60 | """ 61 | if text: 62 | sys.stdout.write(u"{}\n".format(text)) 63 | sys.stdout.flush() 64 | else: 65 | sys.stdout.write(u"\n") 66 | sys.stdout.flush() 67 | 68 | @classmethod 69 | def pl_green(cls, text): 70 | """ 71 | Prints text formatted as green 72 | """ 73 | cls.println(cls.green(text)) 74 | 75 | @classmethod 76 | def pl_red(cls, text): 77 | """ 78 | Prints text formatted as red 79 | """ 80 | cls.println(cls.red(text)) 81 | 82 | @classmethod 83 | def pl_orange(cls, text): 84 | """ 85 | Prints text formatted as orange 86 | """ 87 | cls.println(cls.orange(text)) 88 | 89 | 90 | def check_root_privileges(func): 91 | """ 92 | This function checks if the current user is root. 93 | If the user is not root it exits immediately. 94 | """ 95 | @wraps(func) 96 | def do_root_check(*args, **kwargs): 97 | if os.getuid() != 0: 98 | # check if root user 99 | PrettyPrinter.pl_red("Root privileges are required to run this tool") 100 | sys.exit(1) 101 | return func(*args, **kwargs) 102 | return do_root_check 103 | -------------------------------------------------------------------------------- /tests/test_ssh_manager.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from ceph_salt.core import SshKeyManager 3 | 4 | 5 | class SshManagerTest(unittest.TestCase): 6 | @classmethod 7 | def setUpClass(cls): 8 | cls.priv_key, cls.pub_key = SshKeyManager.generate_key_pair() 9 | cls.priv_key2, cls.pub_key2 = SshKeyManager.generate_key_pair() 10 | 11 | def test_generate_key_pair(self): 12 | try: 13 | SshKeyManager.check_keys(self.priv_key, self.pub_key) 14 | except Exception: # pylint: disable=broad-except 15 | self.fail('SshKeyManager.check_keys raised an Exception') 16 | 17 | def test_key_check_error(self): 18 | with self.assertRaises(Exception) as ctx: 19 | SshKeyManager.check_keys(self.priv_key2, self.pub_key) 20 | self.assertEqual(str(ctx.exception), 'key pair does not match') 21 | 22 | def test_key_check_error2(self): 23 | with self.assertRaises(Exception) as ctx: 24 | SshKeyManager.check_keys(self.priv_key, None) 25 | self.assertEqual(str(ctx.exception), 'key pair does not match') 26 | 27 | def test_key_check_error3(self): 28 | with self.assertRaises(Exception) as ctx: 29 | SshKeyManager.check_keys(None, None) 30 | self.assertEqual(str(ctx.exception), 'invalid private key') 31 | 32 | def test_invalid_private_key(self): 33 | with self.assertRaises(Exception) as ctx: 34 | SshKeyManager.check_keys("invalid key", None) 35 | self.assertEqual(str(ctx.exception), 'invalid private key') 36 | 37 | def test_invalid_private_key2(self): 38 | with self.assertRaises(Exception) as ctx: 39 | SshKeyManager.check_keys(self.pub_key, None) 40 | self.assertEqual(str(ctx.exception), 'invalid private key') 41 | 42 | def test_private_key_not_set(self): 43 | with self.assertRaises(Exception) as ctx: 44 | SshKeyManager.check_private_key(None, None) 45 | self.assertEqual(str(ctx.exception), 'no private key set') 46 | 47 | def test_public_key_not_set(self): 48 | with self.assertRaises(Exception) as ctx: 49 | SshKeyManager.check_public_key(None, None) 50 | self.assertEqual(str(ctx.exception), 'no public key set') 51 | 52 | def test_private_key_does_not_match(self): 53 | with self.assertRaises(Exception) as ctx: 54 | SshKeyManager.check_public_key(None, self.pub_key) 55 | self.assertEqual(str(ctx.exception), 'private key does not match') 56 | 57 | def test_private_key_does_not_match2(self): 58 | with self.assertRaises(Exception) as ctx: 59 | SshKeyManager.check_public_key(self.priv_key2, self.pub_key) 60 | self.assertEqual(str(ctx.exception), 'private key does not match') 61 | 62 | def test_public_key_does_not_match(self): 63 | with self.assertRaises(Exception) as ctx: 64 | SshKeyManager.check_private_key(self.priv_key, None) 65 | self.assertEqual(str(ctx.exception), 'public key does not match') 66 | 67 | def test_public_key_does_not_match2(self): 68 | with self.assertRaises(Exception) as ctx: 69 | SshKeyManager.check_private_key(self.priv_key2, self.pub_key) 70 | self.assertEqual(str(ctx.exception), 'public key does not match') 71 | 72 | def test_fingerprint(self): 73 | key = """ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrI0b980egkmfqFQcsYWrqb2TR3QX/dL+\ 74 | HA5UDa0RFLiOW0xh0liHqd02NZ3j4AoQsh6MSanrROAC2g/cYNDeLo/DR3NXTOPsIhwOkGCncFaOkraVZ/+ZoLsh\ 75 | 8FFXPVz761PgbuzUmz5cQ+IAVSMS5YColvPaynLNtsDQqGwdiL9jB411HbiNnC0oiqU4FpPTa7zFq530WxrtLwee\ 76 | 0P8s0ybiomlBY9m+tYNZJypz4lTPfHa9XHWRn5nxFiqiR5yswRRXeZDAEPBXgN9maIC1Rj2mmDVGpr4v3gKf9TBD\ 77 | PBVw2pLCZsH5ol3VJ1/DETsGRMzFubFeTUNOC3MzhhG+V""" 78 | fingerprint = SshKeyManager.key_fingerprint(key) 79 | self.assertEqual(fingerprint, "7f:fe:76:1b:30:a1:54:56:ff:c3:62:a2:19:3f:40:bd") 80 | -------------------------------------------------------------------------------- /ceph-salt.spec: -------------------------------------------------------------------------------- 1 | # 2 | # spec file for package ceph-salt 3 | # 4 | # Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. 5 | # 6 | # All modifications and additions to the file contributed by third parties 7 | # remain the property of their copyright owners, unless otherwise agreed 8 | # upon. The license for this file, and modifications and additions to the 9 | # file, is the same license as for the pristine package itself (unless the 10 | # license for the pristine package is not an Open Source License, in which 11 | # case the license is the MIT License). An "Open Source License" is a 12 | # license that conforms to the Open Source Definition (Version 1.9) 13 | # published by the Open Source Initiative. 14 | 15 | # Please submit bugfixes or comments via http://bugs.opensuse.org/ 16 | # 17 | 18 | 19 | %if 0%{?el8} || (0%{?fedora} && 0%{?fedora} < 30) 20 | %{python_enable_dependency_generator} 21 | %endif 22 | 23 | Name: ceph-salt 24 | Version: 16.2.5 25 | Release: 1%{?dist} 26 | Summary: CLI tool to deploy Ceph clusters 27 | License: MIT 28 | %if 0%{?suse_version} 29 | Group: System/Management 30 | %endif 31 | URL: https://github.com/ceph/%{name} 32 | Source0: %{url}/archive/v%{version}/%{name}-%{version}.tar.gz 33 | BuildArch: noarch 34 | 35 | %if 0%{?suse_version} 36 | BuildRequires: python-rpm-macros 37 | %else 38 | BuildRequires: python3-devel 39 | %endif 40 | BuildRequires: fdupes 41 | BuildRequires: python3-setuptools 42 | 43 | %if 0%{?suse_version} 44 | Requires: python3-click >= 6.7 45 | Requires: python3-configshell-fb >= 1.1 46 | Requires: python3-distro >= 1.5.0 47 | Requires: python3-pycryptodomex >= 3.9.8 48 | Requires: python3-PyYAML >= 5.1.2 49 | Requires: python3-setuptools 50 | Requires: python3-salt >= 3000 51 | Requires: python3-curses 52 | Requires: python3-ntplib >= 0.3.3 53 | Requires: python3-netaddr 54 | %endif 55 | 56 | Requires: catatonit 57 | Requires: ceph-salt-formula = %{version} 58 | Requires: hostname 59 | Requires: iperf 60 | Requires: iputils 61 | Requires: lsof 62 | Requires: podman 63 | Requires: rsync 64 | Requires: salt-master >= 3000 65 | Requires: sudo 66 | Requires: procps 67 | 68 | Conflicts: deepsea 69 | Conflicts: deepsea-cli 70 | 71 | %description 72 | ceph-salt is a CLI tool for deploying Ceph clusters starting from version 73 | Octopus. 74 | 75 | 76 | %prep 77 | %autosetup -n %{name}-%{version} -p1 78 | 79 | 80 | %build 81 | %py3_build 82 | 83 | 84 | %install 85 | %py3_install 86 | %fdupes %{buildroot}%{python3_sitelib} 87 | install -m 0755 -d %{buildroot}/%{_datadir}/%{name} 88 | 89 | # ceph-salt-formula installation 90 | %define fname ceph-salt 91 | %define fdir %{_datadir}/salt-formulas 92 | 93 | mkdir -p %{buildroot}%{fdir}/states/ 94 | mkdir -p %{buildroot}%{fdir}/metadata/%{fname}/ 95 | cp -R ceph-salt-formula/salt/* %{buildroot}%{fdir}/states/ 96 | cp ceph-salt-formula/metadata/* %{buildroot}%{fdir}/metadata/%{fname}/ 97 | 98 | # manpage 99 | mkdir -p %{buildroot}%{_mandir}/man8/ 100 | cp ceph-salt.8 %{buildroot}%{_mandir}/man8/ceph-salt.8 101 | 102 | 103 | %files 104 | %license LICENSE 105 | %doc CHANGELOG.md README.md 106 | %doc %{_mandir}/man8/ceph-salt.8* 107 | %{python3_sitelib}/ceph_salt*/ 108 | %{_bindir}/%{name} 109 | %dir %{_datadir}/%{name} 110 | 111 | 112 | %package -n ceph-salt-formula 113 | Summary: Ceph Salt Formula 114 | Group: System/Management 115 | 116 | %if ! (0%{?sle_version:1} && 0%{?sle_version} < 150100) 117 | Requires(pre): salt-formulas-configuration 118 | %else 119 | Requires(pre): salt-master 120 | %endif 121 | 122 | 123 | %description -n ceph-salt-formula 124 | Salt Formula to deploy Ceph clusters. 125 | 126 | 127 | %post -n ceph-salt-formula 128 | # This is needed in order for the network.iperf runner to work 129 | salt-run --log-level warning saltutil.sync_runners || : 130 | 131 | %files -n ceph-salt-formula 132 | %defattr(-,root,root,-) 133 | %license LICENSE 134 | %doc README.md 135 | %dir %attr(0755, root, salt) %{fdir}/ 136 | %dir %attr(0755, root, salt) %{fdir}/states/ 137 | %dir %attr(0755, root, salt) %{fdir}/metadata/ 138 | %{fdir}/states/ 139 | %{fdir}/metadata/ 140 | 141 | 142 | %changelog 143 | 144 | -------------------------------------------------------------------------------- /tests/test_pillar_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ceph_salt.salt_utils import PillarManager 3 | from . import SaltMockTestCase 4 | 5 | 6 | class PillarManagerTest(SaltMockTestCase): 7 | 8 | def tearDown(self): 9 | super(PillarManagerTest, self).tearDown() 10 | PillarManager.reload() 11 | 12 | def test_pillar_set(self): 13 | PillarManager.set('ceph-salt:test:enabled', True) 14 | file_path = os.path.join(self.pillar_fs_path(), PillarManager.PILLAR_FILE) 15 | self.assertYamlEqual(file_path, {'ceph-salt': {'test': {'enabled': True}}}) 16 | 17 | def test_pillar_get(self): 18 | PillarManager.set('ceph-salt:test', 'some text') 19 | val = PillarManager.get('ceph-salt:test') 20 | self.assertEqual(val, 'some text') 21 | 22 | def test_pillar_reset(self): 23 | PillarManager.set('ceph-salt:test', 'some text') 24 | val = PillarManager.get('ceph-salt:test') 25 | self.assertEqual(val, 'some text') 26 | PillarManager.reset('ceph-salt:test') 27 | val = PillarManager.get('ceph-salt:test') 28 | self.assertIsNone(val) 29 | 30 | def test_pillar_installed_no_top(self): 31 | self.fs.remove_object('/srv/pillar/ceph-salt.sls') 32 | self.assertFalse(PillarManager.pillar_installed()) 33 | self.fs.create_file('/srv/pillar/ceph-salt.sls') 34 | 35 | def test_pillar_installed_no_top2(self): 36 | self.assertFalse(PillarManager.pillar_installed()) 37 | 38 | def test_pillar_installed_top_with_jinja(self): 39 | self.fs.remove_object('/srv/pillar/ceph-salt.sls') 40 | self.fs.create_file('/srv/pillar/top.sls', contents='{% set x = 2 %}') 41 | self.assertFalse(PillarManager.pillar_installed()) 42 | self.fs.create_file('/srv/pillar/ceph-salt.sls') 43 | self.fs.remove_object('/srv/pillar/top.sls') 44 | 45 | def test_pillar_installed_top_with_jinja2(self): 46 | self.fs.create_file('/srv/pillar/top.sls', contents=''' 47 | {% set x = 2 %}' 48 | base: 49 | {% include 'ceph-salt-top.sls' %} 50 | ''') 51 | self.fs.create_file('/srv/pillar/ceph-salt-top.sls', 52 | contents='''{% import_yaml "ceph-salt.sls" as ceph_salt %} 53 | {% set ceph_salt_minions = ceph_salt.get('ceph-salt', {}).get('minions', {}).get('all', []) %} 54 | {% if ceph_salt_minions %} 55 | {{ ceph_salt_minions|join(',') }}: 56 | - match: list 57 | - ceph-salt 58 | {% endif %} 59 | ''') 60 | self.assertTrue(PillarManager.pillar_installed()) 61 | self.fs.remove_object('/srv/pillar/top.sls') 62 | self.fs.remove_object('/srv/pillar/ceph-salt-top.sls') 63 | 64 | def test_pillar_installed_top_without_base(self): 65 | self.fs.remove_object('/srv/pillar/ceph-salt.sls') 66 | self.fs.create_file('/srv/pillar/top.sls', contents='') 67 | self.assertFalse(PillarManager.pillar_installed()) 68 | self.fs.create_file('/srv/pillar/ceph-salt.sls') 69 | self.fs.remove_object('/srv/pillar/top.sls') 70 | 71 | def test_pillar_installed_top_without_ceph_salt(self): 72 | self.fs.create_file('/srv/pillar/top.sls', contents=''' 73 | base: 74 | '*': [] 75 | ''') 76 | self.assertFalse(PillarManager.pillar_installed()) 77 | self.fs.remove_object('/srv/pillar/top.sls') 78 | 79 | def test_pillar_installed(self): 80 | self.fs.create_file('/srv/pillar/top.sls', contents=''' 81 | base: 82 | {% include 'ceph-salt-top.sls' %} 83 | ''') 84 | self.fs.create_file('/srv/pillar/ceph-salt-top.sls', 85 | contents='''{% import_yaml "ceph-salt.sls" as ceph_salt %} 86 | {% set ceph_salt_minions = ceph_salt.get('ceph-salt', {}).get('minions', {}).get('all', []) %} 87 | {% if ceph_salt_minions %} 88 | {{ ceph_salt_minions|join(',') }}: 89 | - match: list 90 | - ceph-salt 91 | {% endif %} 92 | ''') 93 | self.assertTrue(PillarManager.pillar_installed()) 94 | self.fs.remove_object('/srv/pillar/top.sls') 95 | self.fs.remove_object('/srv/pillar/ceph-salt-top.sls') 96 | 97 | def test_pillar_install(self): 98 | self.fs.remove_object('/srv/pillar/ceph-salt.sls') 99 | self.assertFalse(PillarManager.pillar_installed()) 100 | PillarManager.install_pillar() 101 | self.assertTrue(PillarManager.pillar_installed()) 102 | 103 | def test_pillar_install2(self): 104 | self.fs.create_file('/srv/pillar/top.sls', contents=''' 105 | base: 106 | '*': [] 107 | ''') 108 | self.assertFalse(PillarManager.pillar_installed()) 109 | PillarManager.install_pillar() 110 | self.assertTrue(PillarManager.pillar_installed()) 111 | self.fs.remove_object('/srv/pillar/top.sls') 112 | 113 | def test_pillar_install3(self): 114 | self.fs.remove_object('/srv/pillar/ceph-salt.sls') 115 | self.fs.create_file('/srv/pillar/top.sls', contents='') 116 | self.assertFalse(PillarManager.pillar_installed()) 117 | PillarManager.install_pillar() 118 | self.assertTrue(PillarManager.pillar_installed()) 119 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ceph-salt - Deploy Ceph clusters using cephadm
[![Build Status](https://travis-ci.com/ceph/ceph-salt.svg?branch=master)](https://travis-ci.com/ceph/ceph-salt) [![codecov](https://codecov.io/gh/ceph/ceph-salt/branch/master/graph/badge.svg)](https://codecov.io/gh/ceph/ceph-salt) 2 | 3 | This project provides tools for deploying [Ceph][ceph] clusters managed by 4 | [cephadm][cephadm] using [Salt][salt]. It delivers missing pieces to fully 5 | manage a Ceph cluster with cephadm: 6 | 7 | - OS management (performing OS updates, ntp, tuning) 8 | - Install required RPM dependencies 9 | - Bootstrap cephadm 10 | - Enhanced bootstrapping by defining roles for Salt minions 11 | 12 | # Components 13 | 14 | This repository contains two components: 15 | 16 | 1. `ceph-salt-formula` is a Salt Formula using Salt Highstates to manage Ceph 17 | minions. 18 | 2. `ceph-salt` is a CLI tool to manage the Salt Formula. 19 | 20 | # Architecture 21 | 22 | ![](_images/architecture.png) 23 | 24 | # Setup 25 | 26 | In order to use `ceph-salt`, you need a working Salt cluster and minion IDs 27 | must be resolvable to IP addresses (e.g. `host `). 28 | 29 | Now, install `ceph-salt` on your Salt Master from the openSUSE 30 | repositories: 31 | 32 | ``` 33 | zypper in ceph-salt 34 | ``` 35 | 36 | Afterwards, reload the salt-master daemon: 37 | 38 | ``` 39 | systemctl restart salt-master 40 | salt \* saltutil.sync_all 41 | ``` 42 | 43 | # Usage 44 | 45 | To deploy a Ceph cluster, first run `config` to start the configuration shell to 46 | set the initial deployment of your cluster: 47 | 48 | ``` 49 | ceph-salt config 50 | ``` 51 | 52 | This opens a shell where you can manipulate ceph-salt's configuration. Each 53 | configuration option is present under a configuration group. You can navigate 54 | through the groups and options using the familiar `ls` and `cd` commands 55 | similar to a regular shell. In each path you can type `help` to see the 56 | available commands. Different options might have different commands available. 57 | 58 | First step of configuration is to add the salt-minions that should be managed 59 | by`ceph-salt`. 60 | The `add` command under `/ceph_cluster/minions` option supports autocomplete 61 | and glob expressions: 62 | 63 | ``` 64 | /ceph_cluster/minions add * 65 | ``` 66 | 67 | Then we must specify which minions will be used to deploy Ceph. 68 | Those minions will be Ceph nodes controlled by cephadm: 69 | 70 | ``` 71 | /ceph_cluster/roles/cephadm add * 72 | ``` 73 | 74 | And which of them will have admin "keyring" installed: 75 | 76 | ``` 77 | /ceph_cluster/roles/admin add * 78 | ``` 79 | 80 | Next step is to specify which minion should be used to run `cephadm bootstrap ...`. 81 | This minion will be the Ceph cluster's first Mon and Mgr. 82 | 83 | ``` 84 | /ceph_cluster/roles/bootstrap set node1.test.com 85 | ``` 86 | 87 | Now we need to set the SSH key pair to be used by the Ceph orchestrator. 88 | The SSH key can be generated by ceph-salt with the following command: 89 | 90 | ``` 91 | /ssh generate 92 | ``` 93 | 94 | Alternatively, you can import existing keys by providing the path to the 95 | private and public key files: 96 | 97 | ``` 98 | /ssh/private_key import /root/.ssh/id_rsa 99 | /ssh/public_key import /root/.ssh/id_rsa.pub 100 | ``` 101 | 102 | (Just be aware that ceph-salt does not support SSH private keys that are protected 103 | with a password) 104 | 105 | In the typical ("storage appliance") case, one node of the cluster (typically 106 | the "admin node") will be designated as the "time server host". This node will 107 | sync its clock against one or more external servers (e.g. "pool.ntp.org"), and 108 | all other nodes in the cluster will sync their clocks against this node: 109 | 110 | ``` 111 | /time_server/servers add 112 | /time_server/external_servers add pool.ntp.org 113 | ``` 114 | 115 | Ceph-salt can also configure all nodes to sync against an arbitrary host - this 116 | can be useful when setting up a cluster at a site that already has a single 117 | source of time: 118 | 119 | ``` 120 | /time_server/servers add 121 | ``` 122 | 123 | (In this case, the on-site time server is assumed to be already configured and 124 | ceph-salt will make no attempt to manage it.) 125 | 126 | ceph-salt can also be told not to touch the time sync configuration: 127 | 128 | ``` 129 | /time_server disable 130 | ``` 131 | 132 | In this case, it is up to the user to ensure that some form of time syncing is 133 | configured and running on all machines in the cluster before triggering 134 | `ceph-salt apply`. This is because `cephadm` will refuse to run without it. 135 | 136 | Finally we need to set the Ceph container image path: 137 | 138 | ``` 139 | /cephadm_bootstrap/ceph_image_path set registry.opensuse.org/filesystems/ceph/octopus/images/ceph/ceph 140 | ``` 141 | 142 | Afterwards, you can exit the `ceph-salt` configuration shell by typing `exit` 143 | or pressing `[Ctrl]+[d]`. Now use the `apply` command to start the 144 | `ceph-salt-formula` and execute the deployment: 145 | 146 | ``` 147 | ceph-salt apply 148 | ``` 149 | 150 | Check man page for a full list of available commands: 151 | 152 | ``` 153 | man ceph-salt 154 | ``` 155 | 156 | [ceph]: https://ceph.io/ 157 | [salt]: https://www.saltstack.com/ 158 | [cephadm]: https://docs.ceph.com/en/octopus/cephadm/ 159 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/ceph-salt/apply/cephbootstrap.sls: -------------------------------------------------------------------------------- 1 | {% import 'macros.yml' as macros %} 2 | 3 | {% if grains['id'] == pillar['ceph-salt'].get('bootstrap_minion') and pillar['ceph-salt'].get('execution', {}).get('deployed') != True %} 4 | 5 | {{ macros.begin_stage('Prepare to bootstrap the Ceph cluster') }} 6 | 7 | /var/log/ceph: 8 | file.directory: 9 | - user: ceph 10 | - group: ceph 11 | - mode: '0770' 12 | - makedirs: True 13 | - failhard: True 14 | 15 | {% set auth = pillar['ceph-salt'].get('container', {}).get('auth', {}) %} 16 | 17 | {% if auth %} 18 | 19 | {{ macros.begin_step('Login into registry') }} 20 | 21 | login into registry: 22 | cmd.run: 23 | - name: | 24 | cephadm registry-login \ 25 | --registry-json /tmp/ceph-salt-registry-json 26 | - failhard: True 27 | 28 | {{ macros.end_step('Login into registry') }} 29 | 30 | {% endif %} 31 | 32 | {{ macros.begin_step('Download ceph container image') }} 33 | download ceph container image: 34 | cmd.run: 35 | - name: | 36 | cephadm --image {{ pillar['ceph-salt']['container']['images']['ceph'] }} pull 37 | - failhard: True 38 | {{ macros.end_step('Download ceph container image') }} 39 | 40 | {{ macros.end_stage('Prepare to bootstrap the Ceph cluster') }} 41 | 42 | {{ macros.begin_stage('Bootstrap the Ceph cluster') }} 43 | 44 | {% set bootstrap_ceph_conf = pillar['ceph-salt'].get('bootstrap_ceph_conf', {}) %} 45 | {% set bootstrap_ceph_conf_tmpfile = "/tmp/bootstrap-ceph.conf" %} 46 | {% set bootstrap_spec_yaml_tmpfile = "/tmp/bootstrap-spec.yaml" %} 47 | {% set my_hostname = salt['ceph_salt.hostname']() %} 48 | 49 | create static bootstrap yaml: 50 | cmd.run: 51 | - name: | 52 | echo -en "" > {{ bootstrap_spec_yaml_tmpfile }} 53 | echo -en "service_type: mgr\n" >> {{ bootstrap_spec_yaml_tmpfile }} 54 | echo -en "service_name: mgr\n" >> {{ bootstrap_spec_yaml_tmpfile }} 55 | echo -en "placement:\n" >> {{ bootstrap_spec_yaml_tmpfile }} 56 | echo -en " hosts:\n" >> {{ bootstrap_spec_yaml_tmpfile }} 57 | echo -en " - '{{ my_hostname }}'\n" >> {{ bootstrap_spec_yaml_tmpfile }} 58 | >> foo 59 | echo -en "---\n" >> {{ bootstrap_spec_yaml_tmpfile }} 60 | echo -en "service_type: mon\n" >> {{ bootstrap_spec_yaml_tmpfile }} 61 | echo -en "service_name: mon\n" >> {{ bootstrap_spec_yaml_tmpfile }} 62 | echo -en "placement:\n" >> {{ bootstrap_spec_yaml_tmpfile }} 63 | echo -en " hosts:\n" >> {{ bootstrap_spec_yaml_tmpfile }} 64 | echo -en " - '{{ my_hostname }}:{{ pillar['ceph-salt']['bootstrap_mon_ip'] }}'\n" >> {{ bootstrap_spec_yaml_tmpfile }} 65 | 66 | create bootstrap ceph conf: 67 | cmd.run: 68 | - name: | 69 | echo -en "" > {{ bootstrap_ceph_conf_tmpfile }} 70 | {% for section, settings in bootstrap_ceph_conf.items() %} 71 | echo -en "[{{ section }}]\n" >> {{ bootstrap_ceph_conf_tmpfile }} 72 | {% for setting, value in settings.items() %} 73 | echo -en " {{ setting }} = {{ value }}\n" >> {{ bootstrap_ceph_conf_tmpfile }} 74 | {% endfor %} 75 | {% endfor %} 76 | - failhard: True 77 | 78 | {{ macros.begin_step('Run "cephadm bootstrap"') }} 79 | 80 | {% set dashboard_username = pillar['ceph-salt']['dashboard']['username'] %} 81 | {% set dashboard_password = pillar['ceph-salt']['dashboard']['password'] %} 82 | {% set auth = pillar['ceph-salt'].get('container', {}).get('auth', {}) %} 83 | 84 | {# --mon-ip is still required, even though we're also putting the Mon IP #} 85 | {# directly in the placement YAML (see https://tracker.ceph.com/issues/46782) #} 86 | run cephadm bootstrap: 87 | cmd.run: 88 | - name: | 89 | CEPHADM_IMAGE={{ pillar['ceph-salt']['container']['images']['ceph'] }} \ 90 | cephadm --verbose bootstrap \ 91 | --mon-ip {{ pillar['ceph-salt']['bootstrap_mon_ip'] }} \ 92 | --allow-fqdn-hostname \ 93 | --apply-spec {{ bootstrap_spec_yaml_tmpfile }} \ 94 | --config {{ bootstrap_ceph_conf_tmpfile }} \ 95 | --container-init \ 96 | {%- if not pillar['ceph-salt']['dashboard']['password_update_required'] %} 97 | --dashboard-password-noupdate \ 98 | {%- endif %} 99 | --initial-dashboard-password {{ dashboard_password }} \ 100 | --initial-dashboard-user {{ dashboard_username }} \ 101 | --output-config /etc/ceph/ceph.conf \ 102 | --output-keyring /tmp/ceph.client.admin.keyring \ 103 | {%- if auth %} 104 | --registry-json /tmp/ceph-salt-registry-json \ 105 | {%- endif %} 106 | --skip-monitoring-stack \ 107 | --skip-prepare-host \ 108 | --skip-pull \ 109 | --ssh-private-key {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa \ 110 | --ssh-public-key {{ salt['user.info']('cephadm').home }}/.ssh/id_rsa.pub \ 111 | --ssh-user cephadm \ 112 | {%- for arg, value in pillar['ceph-salt'].get('bootstrap_arguments', {}).items() %} 113 | --{{ arg }} {{ value if value is not none else '' }} \ 114 | {%- endfor %} 115 | > /var/log/ceph/cephadm.out 2>&1 116 | - env: 117 | - NOTIFY_SOCKET: '' 118 | - creates: 119 | - /etc/ceph/ceph.conf 120 | - failhard: True 121 | 122 | {{ macros.end_step('Run "cephadm bootstrap"') }} 123 | 124 | copy ceph.conf and keyring to any admin node: 125 | ceph_orch.copy_ceph_conf_and_keyring_to_any_admin: 126 | - failhard: True 127 | 128 | remove temporary keyring: 129 | file.absent: 130 | - name: /tmp/ceph.client.admin.keyring 131 | - failhard: True 132 | 133 | {{ macros.end_stage('Bootstrap the Ceph cluster') }} 134 | 135 | {% endif %} 136 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/_modules/ceph_salt.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import json 3 | import socket 4 | import time 5 | 6 | import logging 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | def _send_event(tag, data): 12 | __salt__['event.send'](tag, data=data) 13 | return { 14 | 'name': tag, 15 | 'result': True, 16 | 'changes': data, 17 | 'comment': '' 18 | } 19 | 20 | 21 | def begin_stage(name): 22 | return _send_event('ceph-salt/stage/begin', data={'desc': name}) 23 | 24 | 25 | def end_stage(name): 26 | return _send_event('ceph-salt/stage/end', data={'desc': name}) 27 | 28 | 29 | def begin_step(name): 30 | return _send_event('ceph-salt/step/begin', data={'desc': name}) 31 | 32 | 33 | def end_step(name): 34 | return _send_event('ceph-salt/step/end', data={'desc': name}) 35 | 36 | 37 | def ssh(host, cmd, attempts=1): 38 | assert attempts > 0 39 | attempts_count = 0 40 | retry = True 41 | cephadm_home = __salt__['user.info']('cephadm')['home'] 42 | full_cmd = ("ssh -o StrictHostKeyChecking=no " 43 | "-o UserKnownHostsFile=/dev/null " 44 | "-o ConnectTimeout=30 " 45 | "-i {}/.ssh/id_rsa " 46 | "cephadm@{} \"{}\"".format(cephadm_home, host, cmd)) 47 | log.info("ceph_salt.ssh: running SSH command {}".format(full_cmd)) 48 | while retry: 49 | ret = __salt__['cmd.run_all'](full_cmd) 50 | attempts_count += 1 51 | retcode = ret['retcode'] 52 | # 255: Connection closed by remote host 53 | if retcode in [255]: 54 | retry = attempts_count < attempts 55 | if retry: 56 | log.info("Retrying SSH execution after failure with retcode '%s' (%s/%s)", 57 | retcode, attempts_count, attempts - 1) 58 | time.sleep(5) 59 | else: 60 | retry = False 61 | return ret 62 | 63 | 64 | def sudo_rsync(src, dest, ignore_existing): 65 | ignore_existing_option = '--ignore-existing ' if ignore_existing else '' 66 | cephadm_home = __salt__['user.info']('cephadm')['home'] 67 | return __salt__['cmd.run_all']("sudo rsync --rsync-path='sudo rsync' " 68 | "-e 'ssh -o StrictHostKeyChecking=no " 69 | "-o UserKnownHostsFile=/dev/null " 70 | "-o ConnectTimeout=30 " 71 | "-i {}/.ssh/id_rsa' " 72 | "{}{} {} ".format(cephadm_home, ignore_existing_option, src, dest)) 73 | 74 | 75 | def get_remote_grain(host, grain): 76 | """ 77 | Reads remote host grain by accessing '/etc/salt/grains' file directly. 78 | """ 79 | python_script = ''' 80 | import json 81 | import salt.utils.data 82 | import yaml 83 | with open('/etc/salt/grains') as grains_file: 84 | grains = yaml.full_load(grains_file) 85 | val = salt.utils.data.traverse_dict_and_list(grains, '{}') 86 | print(json.dumps({{'local': val}})) 87 | '''.format(grain) 88 | ret = __salt__['ceph_salt.ssh']( 89 | host, 90 | "sudo python3 - < time.time(): 114 | return True 115 | return False 116 | 117 | 118 | def probe_dns(*hostnames): 119 | """ 120 | given a list of hostnames, verify that all can be resolved to IP addresses 121 | """ 122 | ret_status = True 123 | for hostname in hostnames: 124 | log_msg = "probe_dns: attempting to resolve minion hostname ->{}<-".format(hostname) 125 | log.info(log_msg) 126 | try: 127 | socket.gethostbyname(hostname) 128 | except Exception as exc: 129 | log.error(exc) 130 | ret_status = False 131 | if not ret_status: 132 | break 133 | return ret_status 134 | 135 | 136 | def probe_time_sync(): 137 | units = [ 138 | 'chrony.service', # 18.04 (at least) 139 | 'chronyd.service', # el / opensuse 140 | 'systemd-timesyncd.service', 141 | 'ntpd.service', # el7 (at least) 142 | 'ntp.service', # 18.04 (at least) 143 | ] 144 | if not _check_units(units): 145 | log_msg = ('No time sync service is running; checked for: ' 146 | .format(', '.join(units))) 147 | log.warning(log_msg) 148 | return False 149 | return True 150 | 151 | 152 | def _check_units(units): 153 | for unit in units: 154 | (enabled, installed, state) = __check_unit(unit) 155 | if enabled and state == 'running': 156 | log.info('Unit %s is enabled and running' % unit) 157 | return True 158 | return False 159 | 160 | 161 | def __check_unit(unit_name): 162 | # NOTE: we ignore the exit code here because systemctl outputs 163 | # various exit codes based on the state of the service, but the 164 | # string result is more explicit (and sufficient). 165 | enabled = False 166 | installed = False 167 | cmd_ret = __salt__['cmd.run_all']("systemctl is-enabled {}".format(unit_name)) 168 | if cmd_ret['retcode'] == 0: 169 | enabled = True 170 | installed = True 171 | elif cmd_ret['stdout'] and "disabled" in cmd_ret['stdout']: 172 | installed = True 173 | state = 'unknown' 174 | cmd_ret = __salt__['cmd.run_all']("systemctl is-active {}".format(unit_name)) 175 | out = cmd_ret.get('stdout', '').strip() 176 | if out in ['active']: 177 | state = 'running' 178 | elif out in ['inactive']: 179 | state = 'stopped' 180 | elif out in ['failed', 'auto-restart']: 181 | state = 'error' 182 | return (enabled, installed, state) 183 | 184 | 185 | def hostname(): 186 | return socket.gethostname() 187 | 188 | 189 | def ip_address(): 190 | return socket.gethostbyname(socket.gethostname()) 191 | 192 | 193 | def probe_fqdn(): 194 | """ 195 | Returns 'YES' if FQDN environment detected, 'NO' if not detected, or 'FAIL' 196 | if detection was not possible. 197 | """ 198 | retval = hostname() 199 | if not retval: 200 | return 'FAIL' 201 | if '.' in retval: 202 | return 'YES' 203 | return 'NO' 204 | -------------------------------------------------------------------------------- /ceph-salt.8: -------------------------------------------------------------------------------- 1 | \" Man page for ceph-salt 2 | .TH "ceph-salt" "8" "" "" "ceph-salt" 3 | .SH "NAME" 4 | ceph\-salt \- deploy Ceph cluster using cephadm 5 | .SH "SYNOPSIS" 6 | .sp 7 | \fBceph-salt\fP [\fI\-\-global\-opts\fP] \fIcommand\fP 8 | [\fI\-\-command\-opts\fP] [\fIcommand\-arguments\fP] 9 | .sp 10 | \fBceph-salt\fP [\fIcommand\fP] \fB\-\-help\fP 11 | .sp 12 | \fBceph-salt\fP \fB\-\-help\fP 13 | .SH "DESCRIPTION" 14 | .sp 15 | ceph\-salt is a Salt\-based command\-line tool for deploying Ceph clusters 16 | managed by cephadm. 17 | .SH "COMMANDS" 18 | .sp 19 | ceph\-salt provides a number of commands. Each command accepts the options 20 | listed in the \fBGLOBAL OPTIONS\fP section. These options must be specified 21 | before the command name. In addition, many commands have specific options, 22 | which are listed in this section. These command-specific options must be 23 | specified after the name of the command and before any of the command arguments. 24 | .sp 25 | \fBapply\fP [\fIoptions\fP] \fI[minion_id]\fP 26 | .RS 4 27 | Applies a configuration by running ceph-salt formula in order to provision 28 | hosts so they can be used as Ceph cluster nodes. On first execution, a new 29 | Ceph cluster with a single MON and MGR will be deployed. Apply command is 30 | idempotent, so it can be executed multiple times. 31 | .sp 32 | \fB\-n\fP, \fB\-\-non\-interactive\fP 33 | .RS 4 34 | Executes without opening the interactive text-based user interface. 35 | .RE 36 | .sp 37 | \fBminion_id\fP 38 | .RS 4 39 | The minion that should be configured. If not specified, all ceph-salt minions 40 | will be configured. 41 | .RE 42 | .RE 43 | .sp 44 | \fBconfig\fP [\fIpath\fP] \fI[subcommand]\fP 45 | .RS 4 46 | Opens an interactive shell where you can manipulate ceph-salt's configuration. 47 | Each configuration option is present under a configuration group. You can 48 | navigate through the groups and options using the familiar \fBls\fP and 49 | \fBcd\fP commands similar to a regular shell. In each path you can type 50 | \fBhelp\fP to see the available commands. Different options might have 51 | different commands available. If both path and subcommand are provided, the 52 | subcommand will be directly executed without opening the interactive shell. 53 | .sp 54 | \fBpath\fP 55 | .RS 4 56 | Configuration path where subcommand should be executed. 57 | .RE 58 | .sp 59 | \fBsubcommand\fP 60 | .RS 4 61 | Subcommand to execute on configuration path. 62 | .RE 63 | .RE 64 | .RE 65 | .sp 66 | \fBdisengage-safety\fP 67 | .RS 4 68 | Disables safeguards so dangerous operations like purging the whole cluster can 69 | be performed. Safety will be automatically re-enabled after 60 seconds. 70 | .RE 71 | .sp 72 | \fBexport\fP [\fIoptions\fP] 73 | .RS 4 74 | Exports the configuration. 75 | .sp 76 | \fB\-p\fP, \fB\-\-pretty\fP 77 | .RS 4 78 | Pretty-prints JSON ouput. 79 | .RE 80 | .RE 81 | .sp 82 | \fBimport\fP \fIfile\fP 83 | .RS 4 84 | Imports a configuration. 85 | .sp 86 | \fBfile\fP 87 | .RS 4 88 | The file containing the previously exported JSON configuration. 89 | .RE 90 | .RE 91 | .sp 92 | \fBpurge\fP [\fIoptions\fP] 93 | .RS 4 94 | Destroys a Ceph cluster. Before being able to destroy a Ceph cluster, the 95 | \fBdisengage\-safety\fP command must be executed. 96 | .sp 97 | \fB\-n\fP, \fB\-\-non\-interactive\fP 98 | .RS 4 99 | Executes without opening the interactive text-based user interface. 100 | .RE 101 | .sp 102 | \fB\-\-yes\-i\-really\-really\-mean\-it\fP 103 | .RS 4 104 | Additional required confirmation to perform this dangerous operation. 105 | .RE 106 | .RE 107 | .sp 108 | \fBreboot\fP [\fIoptions\fP] \fIminion_id\fP 109 | .RS 4 110 | Reboots hosts if some processes are using deleted files, which may happen after 111 | a system update. If Ceph cluster is already deployed, nodes are rebooted 112 | sequentially in an orchestrated way, otherwise, they are rebooted in parallel. 113 | .sp 114 | \fB\-n\fP, \fB\-\-non\-interactive\fP 115 | .RS 4 116 | Executes without opening the interactive text-based user interface. 117 | .RE 118 | .sp 119 | \fB\-f\fP, \fB\-\-force\fP 120 | .RS 4 121 | Force reboot even if not needed. 122 | .RE 123 | .sp 124 | \fBminion_id\fP 125 | .RS 4 126 | The minion that should be rebooted. If not specified, all ceph-salt minions 127 | will be rebooted. 128 | .RE 129 | .RE 130 | .RE 131 | .sp 132 | \fBstatus\fP [\fIoptions\fP] 133 | .RS 4 134 | Displays ceph-salt status, including information about configuration errors and 135 | the number of hosts managed by cephadm. 136 | .sp 137 | \fB\-n\fP, \fB\-\-no\-color\fP 138 | .RS 4 139 | Output without colors. 140 | .RE 141 | .RE 142 | .sp 143 | \fBstop\fP [\fIoptions\fP] 144 | .RS 4 145 | Stops the Ceph cluster. 146 | .sp 147 | \fB\-n\fP, \fB\-\-non\-interactive\fP 148 | .RS 4 149 | Executes without opening the interactive text-based user interface. 150 | .RE 151 | .sp 152 | \fB\-\-yes\-i\-really\-really\-mean\-it\fP 153 | .RS 4 154 | Additional required confirmation to perform this dangerous operation. 155 | .RE 156 | .RE 157 | .sp 158 | \fBupdate\fP [\fIoptions\fP] \fI[minion_id]\fP 159 | .RS 4 160 | Updates all host operating system packages. 161 | .sp 162 | \fB\-n\fP, \fB\-\-non\-interactive\fP 163 | .RS 4 164 | Executes without opening the interactive text-based user interface. 165 | .RE 166 | .sp 167 | \fB\-r\fP, \fB\-\-reboot\fP 168 | .RS 4 169 | Reboot if, after update, some processes are using deleted files. 170 | .RE 171 | .sp 172 | \fBminion_id\fP 173 | .RS 4 174 | The minion that should be updated. If not specified, all ceph-salt minions will 175 | be updated. 176 | .SH "GLOBAL OPTIONS" 177 | .sp 178 | \fB\-l\fP, \fB\-\-log\-level\fP \fI[info|error|debug|silent]\fP 179 | .RS 4 180 | Specify the log level. 181 | .RE 182 | .sp 183 | \fB\-\-log\-file\fP \fIfile\fP 184 | .RS 4 185 | Specify the log file name. 186 | .SH "EXAMPLES" 187 | .sp 188 | \fBDay 1 - deploy a new Ceph cluster\fP 189 | .RS 4 190 | .sp 191 | $ \fBceph\-salt config\fP 192 | .RS 4 193 | Opens the interactive configuration shell that will be used to configure 194 | cluster. 195 | .RE 196 | .sp 197 | $ \fBceph\-salt status\fP 198 | .RS 4 199 | Checks if the provided configuration is valid. 200 | .RE 201 | .sp 202 | $ \fBceph\-salt update --reboot\fP 203 | .RS 4 204 | Ensures that the cluster is updated with latest patches. 205 | .RE 206 | .sp 207 | $ \fBceph\-salt apply\fP 208 | .RS 4 209 | Deploys a Ceph cluster based on the provided configuration. 210 | .RE 211 | .RE 212 | .sp 213 | \fBDay 2 - add a new host to an existing Ceph cluster\fP 214 | .RS 4 215 | .sp 216 | $ \fBceph\-salt config /ceph_cluster/minions add \fP 217 | .RS 4 218 | .sp 219 | Adds a new minion to the configuration, so it will be managed by ceph-salt. 220 | .RE 221 | .sp 222 | $ \fBceph\-salt config /ceph_cluster/roles/cephadm add \fP 223 | .RS 4 224 | .sp 225 | Assigns the cephadm role to the new minion, so it will be added to the Ceph 226 | cluster. 227 | .RE 228 | .sp 229 | $ \fBceph\-salt apply \fP 230 | .RS 4 231 | Applies configuration on the new minion. 232 | .RE 233 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | extension-pkg-whitelist=fcntl,curses 3 | ignore=CVS,ven,virtualenv,templates 4 | ignore-patterns= 5 | jobs=1 6 | limit-inference-results=100 7 | load-plugins= 8 | persistent=yes 9 | suggestion-mode=yes 10 | unsafe-load-any-extension=no 11 | 12 | 13 | [MESSAGES CONTROL] 14 | confidence= 15 | disable=print-statement, 16 | parameter-unpacking, 17 | unpacking-in-except, 18 | old-raise-syntax, 19 | backtick, 20 | long-suffix, 21 | old-ne-operator, 22 | old-octal-literal, 23 | import-star-module-level, 24 | non-ascii-bytes-literal, 25 | raw-checker-failed, 26 | bad-inline-option, 27 | locally-disabled, 28 | file-ignored, 29 | suppressed-message, 30 | useless-suppression, 31 | deprecated-pragma, 32 | use-symbolic-message-instead, 33 | apply-builtin, 34 | basestring-builtin, 35 | buffer-builtin, 36 | cmp-builtin, 37 | coerce-builtin, 38 | execfile-builtin, 39 | file-builtin, 40 | long-builtin, 41 | raw_input-builtin, 42 | reduce-builtin, 43 | standarderror-builtin, 44 | unicode-builtin, 45 | xrange-builtin, 46 | coerce-method, 47 | delslice-method, 48 | getslice-method, 49 | setslice-method, 50 | no-absolute-import, 51 | old-division, 52 | dict-iter-method, 53 | dict-view-method, 54 | next-method-called, 55 | metaclass-assignment, 56 | indexing-exception, 57 | raising-string, 58 | reload-builtin, 59 | oct-method, 60 | hex-method, 61 | nonzero-method, 62 | cmp-method, 63 | input-builtin, 64 | round-builtin, 65 | intern-builtin, 66 | unichr-builtin, 67 | map-builtin-not-iterating, 68 | zip-builtin-not-iterating, 69 | range-builtin-not-iterating, 70 | filter-builtin-not-iterating, 71 | using-cmp-argument, 72 | eq-without-hash, 73 | div-method, 74 | idiv-method, 75 | rdiv-method, 76 | exception-message-attribute, 77 | invalid-str-codec, 78 | sys-max-int, 79 | bad-python3-import, 80 | deprecated-string-function, 81 | deprecated-str-translate-call, 82 | deprecated-itertools-function, 83 | deprecated-types-field, 84 | next-method-defined, 85 | dict-items-not-iterating, 86 | dict-keys-not-iterating, 87 | dict-values-not-iterating, 88 | deprecated-operator-function, 89 | deprecated-urllib-function, 90 | xreadlines-attribute, 91 | deprecated-sys-function, 92 | exception-escape, 93 | comprehension-escape, 94 | missing-module-docstring, 95 | missing-function-docstring, 96 | missing-class-docstring, 97 | too-many-arguments, 98 | too-many-locals, 99 | too-many-branches, 100 | too-few-public-methods, 101 | too-many-instance-attributes, 102 | too-many-statements, 103 | redefined-builtin, 104 | no-self-use, 105 | too-many-public-methods, 106 | too-many-lines, 107 | too-many-return-statements, 108 | 109 | enable=c-extension-no-member 110 | 111 | 112 | [REPORTS] 113 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 114 | output-format=text 115 | reports=no 116 | score=yes 117 | 118 | 119 | [REFACTORING] 120 | max-nested-blocks=5 121 | never-returning-functions=sys.exit 122 | 123 | 124 | [STRING] 125 | check-str-concat-over-line-jumps=no 126 | 127 | 128 | [TYPECHECK] 129 | contextmanager-decorators=contextlib.contextmanager 130 | generated-members= 131 | ignore-mixin-members=yes 132 | ignore-none=yes 133 | ignore-on-opaque-inference=yes 134 | ignored-classes=optparse.Values,thread._local,_thread._local 135 | ignored-modules=curses 136 | missing-member-hint=yes 137 | missing-member-hint-distance=1 138 | missing-member-max-choices=1 139 | signature-mutators= 140 | 141 | 142 | [LOGGING] 143 | logging-format-style=old 144 | logging-modules=logging 145 | 146 | 147 | [BASIC] 148 | argument-naming-style=snake_case 149 | attr-naming-style=snake_case 150 | bad-names=foo, 151 | bar, 152 | baz, 153 | toto, 154 | tutu, 155 | tata 156 | 157 | class-attribute-naming-style=any 158 | class-naming-style=PascalCase 159 | const-naming-style=UPPER_CASE 160 | docstring-min-length=-1 161 | function-naming-style=snake_case 162 | good-names=i, 163 | j, 164 | k, 165 | v, 166 | ex, 167 | Run, 168 | logger, 169 | os, 170 | _ 171 | 172 | include-naming-hint=no 173 | inlinevar-naming-style=any 174 | method-naming-style=snake_case 175 | module-naming-style=snake_case 176 | name-group= 177 | no-docstring-rgx=^_ 178 | property-classes=abc.abstractproperty 179 | variable-naming-style=snake_case 180 | 181 | 182 | [VARIABLES] 183 | additional-builtins= 184 | allow-global-unused-variables=yes 185 | callbacks=cb_, 186 | _cb 187 | 188 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 189 | ignored-argument-names=_.*|^ignored_|^unused_ 190 | init-import=no 191 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 192 | 193 | 194 | [FORMAT] 195 | expected-line-ending-format= 196 | ignore-long-lines=^\s*(# )??$ 197 | indent-after-paren=4 198 | indent-string=' ' 199 | max-line-length=100 200 | max-module-lines=1000 201 | no-space-check=trailing-comma, 202 | dict-separator 203 | 204 | single-line-class-stmt=no 205 | single-line-if-stmt=no 206 | 207 | 208 | [SPELLING] 209 | 210 | max-spelling-suggestions=4 211 | spelling-dict= 212 | spelling-ignore-words= 213 | spelling-private-dict-file= 214 | spelling-store-unknown-words=no 215 | 216 | 217 | [SIMILARITIES] 218 | ignore-comments=yes 219 | ignore-docstrings=yes 220 | ignore-imports=no 221 | min-similarity-lines=4 222 | 223 | 224 | [MISCELLANEOUS] 225 | notes=XXX, 226 | TODO 227 | 228 | [IMPORTS] 229 | allow-any-import-level= 230 | allow-wildcard-with-all=no 231 | analyse-fallback-blocks=no 232 | deprecated-modules=optparse,tkinter.tix 233 | ext-import-graph= 234 | import-graph= 235 | int-import-graph= 236 | known-standard-library= 237 | known-third-party=enchant 238 | preferred-modules= 239 | 240 | 241 | [CLASSES] 242 | defining-attr-methods=__init__, 243 | __new__, 244 | setUp, 245 | __post_init__ 246 | 247 | exclude-protected=_asdict, 248 | _fields, 249 | _replace, 250 | _source, 251 | _make 252 | 253 | valid-classmethod-first-arg=cls 254 | valid-metaclass-classmethod-first-arg=cls 255 | 256 | 257 | [EXCEPTIONS] 258 | overgeneral-exceptions=BaseException, 259 | Exception 260 | -------------------------------------------------------------------------------- /ceph_salt/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logging.config 3 | import signal 4 | import sys 5 | import time 6 | 7 | import click 8 | import pkg_resources 9 | 10 | from .config_shell import run_config_cmdline, run_config_shell, run_status, run_export, run_import 11 | from .exceptions import CephSaltException 12 | from .logging_utils import LoggingUtil 13 | from .terminal_utils import check_root_privileges, PrettyPrinter as PP 14 | from .execute import CephSaltExecutor, run_disengage_safety, run_purge, run_stop 15 | 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | def ceph_salt_main(): 21 | try: 22 | # pylint: disable=unexpected-keyword-arg,no-value-for-parameter 23 | cli(prog_name='ceph-salt') 24 | except CephSaltException as ex: 25 | logger.exception(ex) 26 | PP.pl_red(str(ex)) 27 | sys.exit(1) 28 | 29 | 30 | @click.group() 31 | @click.option('-l', '--log-level', default='info', 32 | type=click.Choice(["info", "error", "debug", "silent"]), 33 | help="set log level (default: info)") 34 | @click.option('--log-file', default='/var/log/ceph-salt.log', 35 | type=click.Path(dir_okay=False), 36 | help="the file path for the log to be stored") 37 | @click.version_option(pkg_resources.get_distribution('ceph-salt'), message="%(version)s") 38 | @check_root_privileges 39 | def cli(log_level, log_file): 40 | LoggingUtil.setup_logging(log_level, log_file) 41 | 42 | 43 | @cli.command(name='config') 44 | @click.argument('config_args', nargs=-1, type=click.UNPROCESSED, required=False) 45 | def config_shell(config_args): 46 | """ 47 | Start ceph-salt configuration shell 48 | """ 49 | if config_args: 50 | def _quote(text): 51 | if ' ' in text: 52 | return '"{}"'.format(text) 53 | return text 54 | config_args = [_quote(config_arg) for config_arg in config_args] 55 | if not run_config_cmdline(" ".join(config_args)): 56 | sys.exit(1) 57 | else: 58 | if not run_config_shell(): 59 | sys.exit(1) 60 | 61 | 62 | @cli.command(name='status') 63 | @click.option('-n', '--no-color', is_flag=True, default=False, 64 | help='Ouput without colors') 65 | def status(no_color): 66 | """ 67 | Check ceph-salt status 68 | """ 69 | if no_color: 70 | PP.disable_colors() 71 | if not run_status(): 72 | sys.exit(1) 73 | 74 | 75 | @cli.command(name='export') 76 | @click.option('-p', '--pretty', is_flag=True, default=False, 77 | help='Pretty-prints JSON ouput') 78 | def export_config(pretty): 79 | """ 80 | Export configuration 81 | """ 82 | if not run_export(pretty): 83 | sys.exit(1) 84 | 85 | 86 | @cli.command(name='import') 87 | @click.argument('config_file', required=True) 88 | def import_config(config_file): 89 | """ 90 | Import configuration 91 | """ 92 | if not run_import(config_file): 93 | sys.exit(1) 94 | 95 | 96 | def _prompt_proceed(msg, default): 97 | really_want_to = click.prompt( 98 | '{} (y/n):'.format(msg), 99 | type=str, 100 | default=default, 101 | ) 102 | if really_want_to.lower()[0] != 'y': 103 | raise click.Abort() 104 | 105 | 106 | @cli.command(name='apply') 107 | @click.option('-n', '--non-interactive', is_flag=True, default=False, 108 | help='Apply config in non-interactive mode') 109 | @click.argument('minion_id', required=False) 110 | def apply(non_interactive, minion_id): 111 | """ 112 | Apply configuration by running ceph-salt formula 113 | """ 114 | executor = CephSaltExecutor(not non_interactive, minion_id, 115 | 'ceph-salt', {}, _prompt_proceed) 116 | retcode = executor.run() 117 | sys.exit(retcode) 118 | 119 | 120 | @cli.command(name='disengage-safety') 121 | def disengage_safety(): 122 | """ 123 | Disable safety so dangerous operations like purging the whole cluster can be performed 124 | """ 125 | retcode = run_disengage_safety() 126 | sys.exit(retcode) 127 | 128 | 129 | @cli.command(name='purge') 130 | @click.option('-n', '--non-interactive', is_flag=True, default=False, 131 | help='Destroy ceph cluster in non-interactive mode') 132 | @click.option('--yes-i-really-really-mean-it', is_flag=True, default=False, 133 | help='Confirm I really want to perform this dangerous operation') 134 | def purge(non_interactive, yes_i_really_really_mean_it): 135 | """ 136 | Destroy ceph cluster 137 | """ 138 | retcode = run_purge(non_interactive, yes_i_really_really_mean_it, _prompt_proceed) 139 | sys.exit(retcode) 140 | 141 | 142 | @cli.command(name='update') 143 | @click.option('-n', '--non-interactive', is_flag=True, default=False, 144 | help='Apply config in non-interactive mode') 145 | @click.option('-r', '--reboot', is_flag=True, default=False, 146 | help='Reboot if needed') 147 | @click.argument('minion_id', required=False) 148 | def update(non_interactive, reboot, minion_id): 149 | """ 150 | Update all packages 151 | """ 152 | executor = CephSaltExecutor(not non_interactive, minion_id, 153 | 'ceph-salt.update', { 154 | 'ceph-salt': { 155 | 'execution': { 156 | 'reboot-if-needed': reboot 157 | } 158 | } 159 | }, _prompt_proceed) 160 | retcode = executor.run() 161 | sys.exit(retcode) 162 | 163 | 164 | @cli.command(name='reboot') 165 | @click.option('-n', '--non-interactive', is_flag=True, default=False, 166 | help='Reboot in non-interactive mode') 167 | @click.option('-f', '--force', is_flag=True, default=False, 168 | help='Force reboot even if not needed') 169 | @click.argument('minion_id', required=False) 170 | def reboot_cmd(non_interactive, force, minion_id): 171 | """ 172 | Reboot hosts if needed 173 | """ 174 | executor = CephSaltExecutor(not non_interactive, minion_id, 175 | 'ceph-salt.reboot', { 176 | 'ceph-salt': { 177 | 'force-reboot': force 178 | } 179 | }, _prompt_proceed) 180 | retcode = executor.run() 181 | sys.exit(retcode) 182 | 183 | 184 | @cli.command(name='stop') 185 | @click.option('-n', '--non-interactive', is_flag=True, default=False, 186 | help="Stop ceph cluster in non-interactive mode") 187 | @click.option('--yes-i-really-really-mean-it', is_flag=True, default=False, 188 | help='Confirm I really want to perform this critical operation') 189 | def stop(non_interactive, yes_i_really_really_mean_it): 190 | """ 191 | Stop ceph cluster 192 | """ 193 | retcode = run_stop(non_interactive, yes_i_really_really_mean_it, _prompt_proceed) 194 | sys.exit(retcode) 195 | 196 | 197 | if __name__ == '__main__': 198 | ceph_salt_main() 199 | -------------------------------------------------------------------------------- /ceph_salt/validate/config.py: -------------------------------------------------------------------------------- 1 | import ipaddress 2 | 3 | from ..core import SshKeyManager 4 | from ..salt_utils import PillarManager 5 | 6 | 7 | def validate_config(deployed, ceph_nodes): 8 | """ 9 | :return: Error message if config is invalid, otherwise "None" 10 | """ 11 | all_minions = PillarManager.get('ceph-salt:minions:all', []) 12 | bootstrap_minion = PillarManager.get('ceph-salt:bootstrap_minion') 13 | cephadm_minions = PillarManager.get('ceph-salt:minions:cephadm', []) 14 | if not deployed: 15 | if not bootstrap_minion: 16 | return "No bootstrap minion specified in config" 17 | if bootstrap_minion not in cephadm_minions: 18 | return "Bootstrap minion must have 'cephadm' role" 19 | dashboard_username = PillarManager.get('ceph-salt:dashboard:username') 20 | if not dashboard_username: 21 | return "No dashboard username specified in config" 22 | dashboard_password = PillarManager.get('ceph-salt:dashboard:password') 23 | if not dashboard_password: 24 | return "No dashboard password specified in config" 25 | dashboard_ssl_certificate = PillarManager.get('ceph-salt:dashboard:ssl_certificate') 26 | dashboard_ssl_certificate_key = PillarManager.get('ceph-salt:dashboard:ssl_certificate_key') 27 | if dashboard_ssl_certificate and not dashboard_ssl_certificate_key: 28 | return "Dashboard SSL certificate provided, but no SSL certificate key specified" 29 | if not dashboard_ssl_certificate and dashboard_ssl_certificate_key: 30 | return "Dashboard SSL certificate key provided, but no SSL certificate specified" 31 | if not isinstance(PillarManager.get('ceph-salt:dashboard:password_update_required'), bool): 32 | return "'ceph-salt:dashboard:password_update_required' must be of type Boolean" 33 | bootstrap_mon_ip = PillarManager.get('ceph-salt:bootstrap_mon_ip') 34 | if not bootstrap_mon_ip: 35 | return "No bootstrap Mon IP specified in config" 36 | if ipaddress.ip_address(bootstrap_mon_ip).is_loopback: 37 | return 'Mon IP cannot be the loopback interface IP' 38 | bootstrap_node = ceph_nodes.get(bootstrap_minion) 39 | if bootstrap_node is None: 40 | return "Cannot find minion '{}'".format(bootstrap_minion) 41 | if bootstrap_mon_ip not in bootstrap_node.ipsv4 and \ 42 | bootstrap_mon_ip not in bootstrap_node.ipsv6: 43 | return "Mon IP '{}' is not an IP of the bootstrap minion " \ 44 | "'{}'".format(bootstrap_mon_ip, bootstrap_minion) 45 | ceph_container_image_path = PillarManager.get('ceph-salt:container:images:ceph') 46 | if not ceph_container_image_path: 47 | return "No Ceph container image path specified in config" 48 | if '.' not in ceph_container_image_path.split('/')[0]: 49 | return "A relative image path was given, but only absolute image paths are supported" 50 | 51 | # roles 52 | for cephadm_minion in cephadm_minions: 53 | if cephadm_minion not in all_minions: 54 | return "Minion '{}' has 'cephadm' role but is not a cluster "\ 55 | "minion".format(cephadm_minion) 56 | admin_minions = PillarManager.get('ceph-salt:minions:admin', []) 57 | if not admin_minions: 58 | return "No admin minion specified in config" 59 | for admin_minion in admin_minions: 60 | if admin_minion not in cephadm_minions: 61 | return "Minion '{}' has 'admin' role but not 'cephadm' "\ 62 | "role".format(admin_minion) 63 | latency_minions = PillarManager.get('ceph-salt:minions:latency', []) 64 | throughput_minions = PillarManager.get('ceph-salt:minions:throughput', []) 65 | for latency_minion in latency_minions: 66 | if latency_minion not in cephadm_minions: 67 | return "Minion '{}' has 'latency' role but not 'cephadm' "\ 68 | "role".format(latency_minion) 69 | if latency_minion in throughput_minions: 70 | return "Minion '{}' has both 'latency' and 'throughput' "\ 71 | "roles".format(latency_minion) 72 | for throughput_minion in throughput_minions: 73 | if throughput_minion not in cephadm_minions: 74 | return "Minion '{}' has 'throughput' role but not 'cephadm' "\ 75 | "role".format(throughput_minion) 76 | 77 | # ssh 78 | priv_key = PillarManager.get('ceph-salt:ssh:private_key') 79 | if not priv_key: 80 | return "No SSH private key specified in config" 81 | pub_key = PillarManager.get('ceph-salt:ssh:public_key') 82 | if not pub_key: 83 | return "No SSH public key specified in config" 84 | try: 85 | SshKeyManager.check_keys(priv_key, pub_key) 86 | except Exception: # pylint: disable=broad-except 87 | return "Invalid SSH key pair" 88 | 89 | # time_server 90 | time_server_enabled = PillarManager.get('ceph-salt:time_server:enabled') 91 | if not isinstance(time_server_enabled, bool): 92 | return "'ceph-salt:time_server:enabled' must be of type Boolean" 93 | if time_server_enabled: 94 | time_server_hosts = PillarManager.get('ceph-salt:time_server:server_hosts') 95 | if not time_server_hosts: 96 | return 'No time server host specified in config' 97 | time_server_is_minion = any(tsh in all_minions for tsh in time_server_hosts) 98 | time_server_subnet = PillarManager.get('ceph-salt:time_server:subnet') 99 | not_minion_err = ('Time server is not a minion: {} ' 100 | 'setting will not have any effect') 101 | if time_server_is_minion and not time_server_subnet: 102 | return 'No time server subnet specified in config' 103 | if not time_server_is_minion and time_server_subnet: 104 | return not_minion_err.format('time server subnet') 105 | external_time_servers = PillarManager.get('ceph-salt:time_server:external_time_servers') 106 | if time_server_is_minion and not external_time_servers: 107 | return 'No external time servers specified in config' 108 | if not time_server_is_minion and external_time_servers: 109 | return not_minion_err.format('external time servers') 110 | 111 | # container 112 | auth = PillarManager.get('ceph-salt:container:auth') 113 | if auth: 114 | username = auth.get('username') 115 | password = auth.get('password') 116 | registry = auth.get('registry') 117 | if username or password or registry: 118 | if not username or not password or not registry: 119 | return "Registry auth configuration is incomplete" 120 | registries = PillarManager.get('ceph-salt:container:registries', []) 121 | for reg1 in registries: 122 | for reg2 in registries: 123 | if reg2.get('location') == reg1.get('location') and \ 124 | reg2.get('insecure') != reg1.get('insecure'): 125 | return "Registry '{}' is defined multiple times " \ 126 | "with conflicting 'insecure' setting".format(reg1.get('location')) 127 | 128 | return None 129 | -------------------------------------------------------------------------------- /ceph_salt/salt_event.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import fnmatch 3 | import logging 4 | import threading 5 | 6 | import salt.config 7 | import salt.utils.event 8 | from salt.ext.tornado.ioloop import IOLoop 9 | 10 | # pylint: disable=C0103 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class SaltEvent: 15 | """ 16 | Base class of a salt Event 17 | """ 18 | def __init__(self, raw_event): 19 | self.raw_event = raw_event 20 | self.tag = raw_event['tag'] 21 | self.minion = raw_event['data']['id'] 22 | self.stamp = datetime.datetime.strptime(raw_event['data']['_stamp'], "%Y-%m-%dT%H:%M:%S.%f") 23 | 24 | def __str__(self): 25 | return "[{}] [{}] [{}]".format(self.stamp, self.minion, self.tag) 26 | 27 | 28 | class JobRetEvent(SaltEvent): 29 | """ 30 | Base class of a job return Event 31 | """ 32 | def __init__(self, raw_event): 33 | super().__init__(raw_event) 34 | self.success = raw_event['data']['success'] 35 | 36 | def __str__(self): 37 | return '{} {}'.format(super().__str__(), self.success) 38 | 39 | 40 | class CephSaltEvent(SaltEvent): 41 | """ 42 | Base class of a ceph-salt Event 43 | """ 44 | def __init__(self, raw_event): 45 | super().__init__(raw_event) 46 | self.desc = raw_event['data']['data']['desc'] 47 | 48 | def __str__(self): 49 | return '{} {}'.format(super().__str__(), self.desc) 50 | 51 | 52 | class EventListener: 53 | """ 54 | This class represents a listener object that listens to particular Salt events. 55 | """ 56 | 57 | def handle_ceph_salt_event(self, event: CephSaltEvent): 58 | """Handle generic ceph-salt event 59 | Args: 60 | event (CephSaltEvent): the salt event 61 | """ 62 | 63 | def handle_begin_stage(self, event: CephSaltEvent): 64 | """Handle begin stage ceph-salt event 65 | Args: 66 | event (CephSaltEvent): the salt event 67 | """ 68 | 69 | def handle_warning_stage(self, event: CephSaltEvent): 70 | """Handle warning stage ceph-salt event 71 | Args: 72 | event (CephSaltEvent): the salt event 73 | """ 74 | 75 | def handle_end_stage(self, event: CephSaltEvent): 76 | """Handle end stage ceph-salt event 77 | Args: 78 | event (CephSaltEvent): the salt event 79 | """ 80 | 81 | def handle_begin_step(self, event: CephSaltEvent): 82 | """Handle begin step ceph-salt event 83 | Args: 84 | event (CephSaltEvent): the salt event 85 | """ 86 | 87 | def handle_end_step(self, event: CephSaltEvent): 88 | """Handle end step ceph-salt event 89 | Args: 90 | event (CephSaltEvent): the salt event 91 | """ 92 | 93 | def handle_minion_reboot(self, event: CephSaltEvent): 94 | """Handle minion_reboot ceph-salt event 95 | Args: 96 | event (CephSaltEvent): the salt event 97 | """ 98 | 99 | def handle_minion_start(self, event: SaltEvent): 100 | """Handle minion_start salt event 101 | Args: 102 | event (SaltEvent): the salt event 103 | """ 104 | 105 | def handle_state_apply_return(self, event: JobRetEvent): 106 | """Handle state.apply job return event 107 | Args: 108 | event (JobRetEvent): the salt event 109 | """ 110 | 111 | 112 | class SaltEventProcessor(threading.Thread): 113 | """ 114 | This class implements an execution loop to listen for the Salt event BUS. 115 | """ 116 | def __init__(self, minions): 117 | super(SaltEventProcessor, self).__init__() 118 | self.running = False 119 | self.listeners = [] 120 | self.io_loop = None 121 | self.event = threading.Event() 122 | self.minions = minions 123 | 124 | def add_listener(self, listener): 125 | """Adds an event listener to the listener list 126 | Args: 127 | listener (EventListener): the listener object 128 | """ 129 | self.listeners.append(listener) 130 | 131 | def is_running(self): 132 | """ 133 | Gets the running state of the processor 134 | """ 135 | return self.running 136 | 137 | def start(self): 138 | self.running = True 139 | super(SaltEventProcessor, self).start() 140 | self.event.wait() 141 | 142 | def run(self): 143 | """ 144 | Starts the IOLoop of Salt Event Processor 145 | """ 146 | self.io_loop = IOLoop.current() 147 | self.event.set() 148 | 149 | opts = salt.config.client_config('/etc/salt/master') 150 | stream = salt.utils.event.get_event('master', io_loop=self.io_loop, opts=opts) 151 | stream.set_event_handler(self._handle_event_recv) 152 | 153 | self.io_loop.start() 154 | 155 | def stop(self): 156 | """ 157 | Sets running flag to False 158 | """ 159 | self.running = False 160 | self.io_loop.stop() 161 | self.listeners.clear() 162 | 163 | def _handle_event_recv(self, raw): 164 | """ 165 | Handles the asynchronous reception of raw events 166 | """ 167 | mtag, data = salt.utils.event.SaltEvent.unpack(raw) 168 | self._process({'tag': mtag, 'data': data}) 169 | 170 | def _process(self, event): 171 | """Processes a raw event 172 | 173 | Creates the proper salt event class wrapper and notifies listeners 174 | 175 | Args: 176 | event (dict): the raw event data 177 | """ 178 | logger.debug("Process event -> %s", event) 179 | wrapper = None 180 | if fnmatch.fnmatch(event['tag'], 'ceph-salt/*'): 181 | wrapper = CephSaltEvent(event) 182 | elif event['tag'] == 'minion_start': 183 | wrapper = SaltEvent(event) 184 | elif fnmatch.fnmatch(event['tag'], 'salt/job/*/ret/*'): 185 | if event['data'].get('fun') == 'state.apply': 186 | wrapper = JobRetEvent(event) 187 | if wrapper: 188 | if wrapper.minion not in self.minions: 189 | return 190 | for listener in self.listeners: 191 | if fnmatch.fnmatch(event['tag'], 'ceph-salt/*'): 192 | listener.handle_ceph_salt_event(wrapper) 193 | if event['tag'] == 'ceph-salt/stage/begin': 194 | listener.handle_begin_stage(wrapper) 195 | elif event['tag'] == 'ceph-salt/stage/end': 196 | listener.handle_end_stage(wrapper) 197 | elif event['tag'] == 'ceph-salt/step/begin': 198 | listener.handle_begin_step(wrapper) 199 | elif event['tag'] == 'ceph-salt/step/end': 200 | listener.handle_end_step(wrapper) 201 | elif event['tag'] == 'ceph-salt/minion_reboot': 202 | listener.handle_minion_reboot(wrapper) 203 | elif event['tag'] == 'ceph-salt/stage/warning': 204 | listener.handle_warning_stage(wrapper) 205 | elif event['tag'] == 'minion_start': 206 | listener.handle_minion_start(wrapper) 207 | elif fnmatch.fnmatch(event['tag'], 'salt/job/*/ret/*'): 208 | if event['data'].get('fun') == 'state.apply': 209 | listener.handle_state_apply_return(wrapper) 210 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/_states/ceph_salt.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import logging 3 | import subprocess 4 | import time 5 | 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | def _send_event(tag, data): 11 | __salt__['event.send'](tag, data=data) 12 | return { 13 | 'name': tag, 14 | 'result': True, 15 | 'changes': data, 16 | 'comment': '' 17 | } 18 | 19 | 20 | def begin_stage(name): 21 | return _send_event('ceph-salt/stage/begin', data={'desc': name}) 22 | 23 | 24 | def end_stage(name): 25 | return _send_event('ceph-salt/stage/end', data={'desc': name}) 26 | 27 | 28 | def begin_step(name): 29 | return _send_event('ceph-salt/step/begin', data={'desc': name}) 30 | 31 | 32 | def end_step(name): 33 | return _send_event('ceph-salt/step/end', data={'desc': name}) 34 | 35 | 36 | def set_reboot_needed(name, force=False): 37 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 38 | if force: 39 | needs_reboot = True 40 | else: 41 | if __grains__.get('os_family') == 'Suse': 42 | cmd_ret = __salt__['cmd.run_all']('zypper ps') 43 | needs_reboot = cmd_ret['stdout'].find('No processes using deleted files found') < 0 44 | else: 45 | ret['comment'] = 'Unsupported distribution: Unable to check if reboot is needed' 46 | return ret 47 | reboot_needed_step = "Reboot is not needed" 48 | if needs_reboot: 49 | if __grains__.get('os_family') == 'Suse': 50 | reboot_needed_step = "Reboot is needed because some processes are using deleted files" 51 | else: 52 | reboot_needed_step = "Reboot is needed" 53 | 54 | __salt__['event.send']('ceph-salt/step/start', 55 | data={'desc': reboot_needed_step}) 56 | __salt__['grains.set']('ceph-salt:execution:reboot_needed', needs_reboot) 57 | __salt__['event.send']('ceph-salt/step/end', 58 | data={'desc': reboot_needed_step}) 59 | # Try to guarantee that event reaches master before job finish 60 | time.sleep(5) 61 | 62 | ret['result'] = True 63 | return ret 64 | 65 | 66 | def reboot_if_needed(name): 67 | """ 68 | Requires the following grains to be set: 69 | - ceph-salt:execution:reboot_needed 70 | """ 71 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 72 | needs_reboot = __salt__['grains.get']('ceph-salt:execution:reboot_needed') 73 | if needs_reboot: 74 | is_master = __salt__['service.status']('salt-master') 75 | if is_master: 76 | __salt__['event.send']('ceph-salt/stage/warning', 77 | data={'desc': "Salt master must be rebooted manually"}) 78 | ret['result'] = True 79 | return ret 80 | __salt__['event.send']('ceph-salt/minion_reboot', data={'desc': 'Rebooting...'}) 81 | # Try to guarantee that event reaches master before job finish 82 | time.sleep(5) 83 | __salt__['system.reboot']() 84 | ret['result'] = True 85 | return ret 86 | 87 | 88 | def wait_for_grain(name, grain, hosts, timeout=1800): 89 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 90 | completed_counter = 0 91 | starttime = time.time() 92 | timelimit = starttime + timeout 93 | while completed_counter < len(hosts): 94 | is_timedout = time.time() > timelimit 95 | if is_timedout: 96 | ret['comment'] = 'Timeout value reached.' 97 | return ret 98 | time.sleep(15) 99 | completed_counter = 0 100 | for host in hosts: 101 | grain_value = __salt__['ceph_salt.get_remote_grain'](host, 'ceph-salt:execution:failed') 102 | if grain_value: 103 | ret['comment'] = 'One or more minions failed.' 104 | return ret 105 | grain_value = __salt__['ceph_salt.get_remote_grain'](host, grain) 106 | if grain_value: 107 | completed_counter += 1 108 | logger.info("Waiting for grain '%s' (%s/%s)", grain, completed_counter, len(hosts)) 109 | ret['result'] = True 110 | return ret 111 | 112 | 113 | def wait_for_ancestor_minion_grain(name, grain, if_grain, timeout=36000): 114 | """ 115 | This state will wait for a grain on the minion that appears immediately before 116 | the current minion, on the 'ceph-salt:execution:minions' pillar list. 117 | 118 | Usefull when dealing with sequential operations. 119 | """ 120 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 121 | id = __grains__['id'] 122 | minions = __pillar__['ceph-salt']['execution']['minions'] 123 | if id not in minions: 124 | ret['comment'] = "Unexpected minion: Minion '{}' not in the execution plan".format(id) 125 | return ret 126 | if_grain_value = __salt__['grains.get'](if_grain) 127 | if if_grain_value: 128 | ancestor_minion = None 129 | for i in range(len(minions)): 130 | if i < (len(minions)-1) and minions[i+1] == id: 131 | ancestor_minion = minions[i] 132 | break 133 | if ancestor_minion: 134 | begin_stage("Wait for '{}'".format(ancestor_minion)) 135 | ancestor_minion_ready = False 136 | starttime = time.time() 137 | timelimit = starttime + timeout 138 | while not ancestor_minion_ready: 139 | is_timedout = time.time() > timelimit 140 | if is_timedout: 141 | ret['comment'] = 'Timeout value reached.' 142 | return ret 143 | grain_value = __salt__['ceph_salt.get_remote_grain'](ancestor_minion, 'ceph-salt:execution:failed') 144 | if grain_value: 145 | ret['comment'] = 'Minion {} failed.'.format(ancestor_minion) 146 | return ret 147 | grain_value = __salt__['ceph_salt.get_remote_grain'](ancestor_minion, grain) 148 | if grain_value: 149 | ancestor_minion_ready = True 150 | if not ancestor_minion_ready: 151 | logger.info("Waiting for grain '%s' on '%s'", grain, ancestor_minion) 152 | time.sleep(15) 153 | end_stage("Wait for '{}'".format(ancestor_minion)) 154 | ret['result'] = True 155 | return ret 156 | 157 | 158 | def check_safety(name): 159 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 160 | cmd_ret = __salt__['ceph_salt.is_safety_disengaged']() 161 | if cmd_ret is not True: 162 | ret['comment'] = "Safety is not disengaged. Run 'ceph-salt disengage-safety' to disable protection against dangerous operations." 163 | return ret 164 | ret['result'] = True 165 | return ret 166 | 167 | 168 | def check_fsid(name, formula): 169 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 170 | fsid = __salt__['pillar.get']('ceph-salt:execution:fsid') 171 | if not fsid: 172 | ret['comment'] = "No cluster FSID provided. Ceph cluster FSID " \ 173 | "must be provided via custom Pillar value, e.g.: " \ 174 | "\"salt -G ceph-salt:member state.apply {} " \ 175 | "pillar='{{\"ceph-salt\": {{\"execution\": " \ 176 | "{{\"fsid\": \"$FSID\"}}}}}}'\"".format(formula) 177 | return ret 178 | ret['result'] = True 179 | return ret 180 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/_modules/multi.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Collection of running multiple processes from a minion in parallel, typically 4 | for network related tasks such as ping and iperf. 5 | """ 6 | 7 | from __future__ import absolute_import 8 | import logging 9 | import multiprocessing.dummy 10 | import multiprocessing 11 | import re 12 | import socket 13 | from subprocess import Popen, PIPE 14 | # pylint: disable=import-error 15 | 16 | log = logging.getLogger(__name__) 17 | 18 | try: 19 | from salt.utils.path import which 20 | except ImportError: 21 | from distutils.spawn import which 22 | 23 | try: 24 | # pylint: disable=import-error,3rd-party-module-not-gated,redefined-builtin 25 | from salt.ext.six.moves import range 26 | except ImportError: 27 | logging.error("Could not import salt.ext.six.moves -> range") 28 | 29 | IPERF_PATH = which('iperf3') 30 | 31 | LOCALHOST_NAME = socket.gethostname() 32 | 33 | ''' 34 | multi is the module to call subprocess in minion host 35 | 36 | Ping is a simple test to check if point to point nodes are connected 37 | 38 | CLI Example: 39 | .. code-block:: bash 40 | sudo salt 'node' multi.ping_cmd | 41 | sudo salt 'node' multi.ping | |.... 42 | ''' 43 | 44 | 45 | def _all(func, hosts): 46 | ''' 47 | Internal function that allow function to perform in all hosts 48 | ''' 49 | all_instances = [] 50 | # threads should likely scale with cores or interfaces 51 | cpus = multiprocessing.cpu_count() 52 | threads = 4 * cpus 53 | log.debug('multi._all cpus count={}, thread count={}'.format(cpus, threads)) 54 | pool = multiprocessing.dummy.Pool(threads) 55 | for instance in pool.map(func, hosts): 56 | all_instances.append(instance) 57 | 58 | return all_instances 59 | 60 | 61 | def _summarize_iperf(result): 62 | ''' 63 | Scan the results and summarize for iperf result 64 | ''' 65 | # pylint: disable=invalid-name,unused-variable 66 | host, rc, out, err = result 67 | msg = {} 68 | msg['server'] = host 69 | if rc == 0: 70 | msg['succeeded'] = True 71 | msg['speed'] = out 72 | try: 73 | msg['filter'] = re.match( 74 | r'.*0.00-10.00.*sec\s(.*Bytes)\s+(.*Mbits/sec)', 75 | out, re.DOTALL).group(2) 76 | except AttributeError: 77 | msg['filter'] = '0 Mbits/sec' 78 | msg['failed'] = False 79 | msg['errored'] = False 80 | if rc == 1: 81 | msg['succeeded'] = False 82 | msg['failed'] = True 83 | msg['errored'] = False 84 | if rc == 2: 85 | msg['succeeded'] = False 86 | msg['failed'] = False 87 | msg['errored'] = True 88 | return msg 89 | 90 | 91 | def _summarize_ping(results): 92 | ''' 93 | Scan the results and summarize 94 | ''' 95 | success = [] 96 | failed = [] 97 | errored = [] 98 | slow = [] 99 | avg = [] 100 | for result in results: 101 | # pylint: disable=invalid-name,unused-variable 102 | host, rc, out, err = result 103 | if rc == 0: 104 | success.append(host) 105 | rtt = re.match( 106 | r'.*rtt min/avg/max/mdev = \d+\.?\d+/(\d+\.?\d+)/', 107 | out, re.DOTALL) 108 | if rtt: 109 | avg.append({'avg': float(rtt.group(1)), 'host': host}) 110 | if rc == 1: 111 | failed.append(host) 112 | if rc == 2: 113 | errored.append(host) 114 | 115 | log.debug('multi._summarize_ping average={}'.format(avg)) 116 | 117 | if avg: 118 | avg_sum = sum(i.get('avg') for i in avg) / len(avg) 119 | if len(avg) > 2: 120 | for i in avg: 121 | if (avg_sum * len(avg) / 2) < i.get('avg'): 122 | log.debug('_summarize_ping: slow host = {} avg = {}, s'. 123 | format(i.get('avg'), avg_sum)) 124 | slow.append(i.get('host')) 125 | else: 126 | avg_sum = 0 127 | 128 | msg = {} 129 | msg['succeeded'] = len(success) 130 | if failed: 131 | msg['failed'] = " ".join(failed) 132 | if errored: 133 | msg['errored'] = " ".join(errored) 134 | if slow: 135 | msg['slow'] = " ".join(slow) 136 | msg['avg'] = avg_sum 137 | return msg 138 | 139 | 140 | def iperf(server, cpu, port): 141 | ''' 142 | iperf test to a specific server 143 | 144 | CLI Example: 145 | .. code-block:: bash 146 | sudo salt 'node' multi.iperf | 147 | ''' 148 | log.debug('iperf server ={}'.format(server)) 149 | return _summarize_iperf(iperf_client_cmd(server, cpu, port)) 150 | 151 | 152 | def iperf_client_cmd(server, cpu=0, port=5200): 153 | ''' 154 | Use iperf to test minion to server 155 | 156 | CLI Example: 157 | .. code-block:: bash 158 | salt 'node' multi.iperf_client_cmd 159 | cpu= port= 160 | ''' 161 | if IPERF_PATH is None: 162 | ret = [LOCALHOST_NAME, 2, "0", 163 | "iperf3 not found in path, please install"] 164 | elif not server: 165 | ret = [LOCALHOST_NAME, 2, "0", "Server name is empty"] 166 | else: 167 | cmd = ["/usr/bin/iperf3", "-fm", "-A"+str(cpu), 168 | "-t10", "-c"+server, "-p"+str(port)] 169 | log.debug('iperf_client_cmd: cmd {}'.format(cmd)) 170 | proc = Popen(cmd, stdout=PIPE, stderr=PIPE) 171 | proc.wait() 172 | ret = (server, proc.returncode, proc.stdout.read().decode(), proc.stderr.read().decode()) 173 | return ret 174 | 175 | 176 | def iperf_server_cmd(cpu=0, port=5200): 177 | ''' 178 | Use iperf to test minion to server 179 | 180 | CLI Example: 181 | .. code-block:: bash 182 | salt 'node' multi.iperf_server_cmd 183 | cpu= port= 184 | ''' 185 | if IPERF_PATH is None: 186 | return LOCALHOST_NAME + ": iperf3 not found in path, please install" 187 | iperf_cmd = ["/usr/bin/iperf3", "-s", "-D", "-A"+str(cpu), "-p"+str(port)] 188 | log.debug('iperf_server_cmd: cmd {}'.format(iperf_cmd)) 189 | Popen(iperf_cmd) 190 | # it doesn't report fail so no need to check 191 | return LOCALHOST_NAME + ": iperf3 started at cpu " + str(cpu) + " port " + str(port) + "\n" 192 | 193 | 194 | def kill_iperf_cmd(): 195 | ''' 196 | Clean up all the iperf3 server and clean it. 197 | ''' 198 | kill_cmd = ["/usr/bin/killall", "-9", "iperf3"] 199 | log.debug('kill_iperf_cmd: cmd {}'.format(kill_cmd)) 200 | Popen(kill_cmd) 201 | return True 202 | 203 | 204 | def ping(*hosts): 205 | ''' 206 | Ping a list of hosts and summarize the results 207 | 208 | CLI Example: 209 | .. code-block:: bash 210 | sudo salt 'node' multi.ping | |.... 211 | ''' 212 | # I should be filter all the localhost here? 213 | log.debug('ping hostlist={}'.format(list(hosts))) 214 | results = _all(ping_cmd, list(hosts)) 215 | return _summarize_ping(results) 216 | 217 | 218 | def ping_cmd(host): 219 | ''' 220 | Ping a host with 1 packet and return the result 221 | 222 | CLI Example: 223 | .. code-block:: bash 224 | sudo salt 'node' multi.ping_cmd | 225 | ''' 226 | cmd = ["/usr/bin/ping", "-c1", "-q", "-W1", host] 227 | log.debug('ping_cmd hostname={}'.format(host)) 228 | proc = Popen(cmd, stdout=PIPE, stderr=PIPE) 229 | proc.wait() 230 | return host, proc.returncode, proc.stdout.read().decode(), proc.stderr.read().decode() 231 | 232 | 233 | def jumbo_ping(*hosts): 234 | ''' 235 | Ping a list of hosts and summarize the results 236 | 237 | CLI Example: 238 | .. code-block:: bash 239 | sudo salt 'node' multi.ping | |.... 240 | ''' 241 | # I should be filter all the localhost here? 242 | log.debug('jumbo_ping hostlist={}'.format(list(hosts))) 243 | results = _all(jumbo_ping_cmd, list(hosts)) 244 | return _summarize_ping(results) 245 | 246 | 247 | def jumbo_ping_cmd(host): 248 | ''' 249 | Ping a host with 1 packet and return the result 250 | 251 | CLI Example: 252 | .. code-block:: bash 253 | sudo salt 'node' multi.ping_cmd | 254 | ''' 255 | cmd = ["/usr/bin/ping", "-Mdo", "-s8972", "-c1", "-q", "-W1", host] 256 | log.debug('ping_cmd hostname={}'.format(host)) 257 | proc = Popen(cmd, stdout=PIPE, stderr=PIPE) 258 | proc.wait() 259 | return host, proc.returncode, proc.stdout.read().decode(), proc.stderr.read().decode() 260 | 261 | 262 | def prepare_iperf_server(): 263 | ''' 264 | Create N server base on the total core number of your cpu count 265 | 266 | CLI Example: 267 | .. code-block:: bash 268 | salt 'node' multi.prepare_iperf_server 269 | 270 | ''' 271 | cpus = multiprocessing.cpu_count() 272 | iperf_log = "" 273 | for cpu in range(cpus): 274 | iperf_log += iperf_server_cmd(cpu, 5200+cpu) 275 | return iperf_log 276 | -------------------------------------------------------------------------------- /ceph_salt/core.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import copy 3 | import hashlib 4 | import ipaddress 5 | import logging 6 | import os 7 | 8 | from Cryptodome.PublicKey import RSA 9 | import salt 10 | 11 | from .exceptions import CephNodeHasRolesException 12 | from .salt_utils import GrainsManager, PillarManager, SaltClient 13 | 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | CEPH_SALT_GRAIN_KEY = 'ceph-salt' 19 | 20 | 21 | class CephNode: 22 | def __init__(self, minion_id): 23 | self.minion_id = minion_id 24 | self._ipsv4 = None 25 | self._ipsv6 = None 26 | self._roles = None 27 | self._execution = None 28 | self._public_ip = None 29 | self._subnets = None 30 | self._public_subnet = None 31 | self._os_codename = None 32 | self._ceph_version = None 33 | 34 | @property 35 | def ipsv4(self): 36 | if self._ipsv4 is None: 37 | result = GrainsManager.get_grain(self.minion_id, 'ipv4') 38 | self._ipsv4 = result[self.minion_id] 39 | return self._ipsv4 40 | 41 | @property 42 | def ipsv6(self): 43 | if self._ipsv6 is None: 44 | result = GrainsManager.get_grain(self.minion_id, 'ipv6') 45 | self._ipsv6 = result[self.minion_id] 46 | return self._ipsv6 47 | 48 | @property 49 | def public_ip(self): 50 | def _is_loopback(addr): 51 | return ipaddress.ip_address(addr).is_loopback 52 | 53 | if self._public_ip is None: 54 | result = GrainsManager.get_grain(self.minion_id, 'fqdn_ip4') 55 | _public_ip = result[self.minion_id][0] 56 | if _is_loopback(_public_ip): 57 | logger.debug("fqdn_ipv4 grain is '%s', falling back to ipv4 grain", _public_ip) 58 | result = GrainsManager.get_grain(self.minion_id, 'ipv4') 59 | for addr in result[self.minion_id]: 60 | if not _is_loopback(addr): 61 | _public_ip = addr 62 | break 63 | if _is_loopback(_public_ip): 64 | logger.warning("'%s' public IP is the loopback interface IP ('%s')", 65 | self.minion_id, _public_ip) 66 | self._public_ip = _public_ip 67 | return self._public_ip 68 | 69 | @property 70 | def subnets(self): 71 | if self._subnets is None: 72 | result = SaltClient.local_cmd(self.minion_id, 'network.subnets') 73 | self._subnets = result[self.minion_id] 74 | return self._subnets 75 | 76 | @property 77 | def public_subnet(self): 78 | if self._public_subnet is None: 79 | if self.public_ip and self.subnets: 80 | for subnet in self.subnets: 81 | if salt.utils.network.in_subnet(subnet, self.public_ip): 82 | self._public_subnet = subnet 83 | break 84 | return self._public_subnet 85 | 86 | @property 87 | def roles(self): 88 | if self._roles is None: 89 | roles = PillarManager.get('ceph-salt:minions', {}) 90 | _roles = set() 91 | for role, minions in roles.items(): 92 | if role != 'all' and self.minion_id in minions: 93 | _roles.add(role) 94 | self._roles = _roles 95 | return self._roles 96 | 97 | @property 98 | def execution(self): 99 | if self._execution is None: 100 | result = GrainsManager.get_grain(self.minion_id, CEPH_SALT_GRAIN_KEY) 101 | self._execution = {} 102 | if 'execution' in result[self.minion_id]: 103 | self._execution = result[self.minion_id]['execution'] 104 | return self._execution 105 | 106 | @property 107 | def os_codename(self): 108 | if self._os_codename is None: 109 | result = GrainsManager.get_grain(self.minion_id, 'oscodename') 110 | self._os_codename = result[self.minion_id] 111 | return self._os_codename 112 | 113 | @property 114 | def ceph_version(self): 115 | if self._ceph_version is None: 116 | result = SaltClient.local_cmd(self.minion_id, 'cmd.shell', [ 117 | 'test -e /usr/bin/ceph && ceph --version || echo "Not installed"' 118 | ]) 119 | self._ceph_version = result[self.minion_id] 120 | return self._ceph_version 121 | 122 | def add_role(self, role): 123 | self.roles.add(role) 124 | 125 | def _role_list(self): 126 | role_list = list(self.roles) 127 | role_list.sort() 128 | return role_list 129 | 130 | def _grains_value(self): 131 | return { 132 | 'member': True, 133 | 'execution': self.execution, 134 | 'roles': self._role_list() 135 | } 136 | 137 | def save(self): 138 | GrainsManager.set_grain(self.minion_id, CEPH_SALT_GRAIN_KEY, self._grains_value()) 139 | 140 | 141 | class CephNodeManager: 142 | _ceph_salt_nodes = {} 143 | 144 | @classmethod 145 | def _load(cls): 146 | if not cls._ceph_salt_nodes: 147 | minions = GrainsManager.filter_by(CEPH_SALT_GRAIN_KEY) 148 | cls._ceph_salt_nodes = {minion: CephNode(minion) for minion in minions} 149 | 150 | @classmethod 151 | def save_in_pillar(cls): 152 | minions = [n.minion_id for n in cls._ceph_salt_nodes.values()] 153 | PillarManager.set('ceph-salt:minions:all', minions) 154 | PillarManager.set('ceph-salt:minions:admin', 155 | [n.minion_id for n in cls._ceph_salt_nodes.values() 156 | if 'admin' in n.roles]) 157 | PillarManager.set('ceph-salt:minions:cephadm', 158 | [n.minion_id for n in cls._ceph_salt_nodes.values() 159 | if 'cephadm' in n.roles]) 160 | PillarManager.set('ceph-salt:minions:latency', 161 | [n.minion_id for n in cls._ceph_salt_nodes.values() 162 | if 'latency' in n.roles]) 163 | PillarManager.set('ceph-salt:minions:throughput', 164 | [n.minion_id for n in cls._ceph_salt_nodes.values() 165 | if 'throughput' in n.roles]) 166 | 167 | @classmethod 168 | def ceph_salt_nodes(cls): 169 | cls._load() 170 | return cls._ceph_salt_nodes 171 | 172 | @classmethod 173 | def add_node(cls, minion_id): 174 | cls._load() 175 | node = CephNode(minion_id) 176 | node.save() 177 | cls._ceph_salt_nodes[minion_id] = node 178 | cls.save_in_pillar() 179 | 180 | @classmethod 181 | def remove_node(cls, minion_id): 182 | cls._load() 183 | roles = cls.all_roles(cls._ceph_salt_nodes[minion_id]) 184 | if roles: 185 | raise CephNodeHasRolesException(minion_id, sorted(roles)) 186 | del cls._ceph_salt_nodes[minion_id] 187 | GrainsManager.del_grain(minion_id, CEPH_SALT_GRAIN_KEY) 188 | cls.save_in_pillar() 189 | 190 | @classmethod 191 | def list_all_minions(cls): 192 | return os.listdir(SaltClient.pki_minions_fs_path()) 193 | 194 | @staticmethod 195 | def all_roles(ceph_salt_node): 196 | roles = copy.deepcopy(ceph_salt_node.roles) 197 | bootstrap_minion = PillarManager.get('ceph-salt:bootstrap_minion') 198 | if ceph_salt_node.minion_id == bootstrap_minion: 199 | roles.add('bootstrap') 200 | return roles 201 | 202 | 203 | class SshKeyManager: 204 | @staticmethod 205 | def key_fingerprint(key): 206 | key = base64.b64decode(key.split()[1].encode('ascii')) 207 | fp_plain = hashlib.md5(key).hexdigest() 208 | return ':'.join(a + b for a, b in zip(fp_plain[::2], fp_plain[1::2])) 209 | 210 | @staticmethod 211 | def generate_key_pair(bits=2048): 212 | key = RSA.generate(bits) 213 | private_key = key.exportKey('PEM') 214 | public_key = key.publickey().exportKey('OpenSSH') 215 | return private_key.decode('utf-8'), public_key.decode('utf-8') 216 | 217 | @classmethod 218 | def check_keys(cls, stored_priv_key, stored_pub_key): 219 | try: 220 | key = RSA.import_key(stored_priv_key) 221 | except (ValueError, IndexError, TypeError): 222 | raise Exception('invalid private key') 223 | 224 | if not key.has_private(): 225 | raise Exception('invalid private key') 226 | 227 | pub_key = key.publickey().exportKey('OpenSSH').decode('utf-8') 228 | if not stored_pub_key or pub_key.split()[1] != stored_pub_key.split()[1]: 229 | raise Exception('key pair does not match') 230 | 231 | @classmethod 232 | def check_public_key(cls, stored_priv_key, stored_pub_key): 233 | if not stored_pub_key: 234 | raise Exception('no public key set') 235 | if not stored_priv_key: 236 | raise Exception('private key does not match') 237 | try: 238 | cls.check_keys(stored_priv_key, stored_pub_key) 239 | except Exception as ex: 240 | if str(ex) == 'key pair does not match': 241 | ex = Exception('private key does not match') 242 | raise ex 243 | 244 | @classmethod 245 | def check_private_key(cls, stored_priv_key, stored_pub_key): 246 | if not stored_priv_key: 247 | raise Exception('no private key set') 248 | if not stored_pub_key: 249 | raise Exception('public key does not match') 250 | try: 251 | cls.check_keys(stored_priv_key, stored_pub_key) 252 | except Exception as ex: 253 | if str(ex) == 'key pair does not match': 254 | ex = Exception('public key does not match') 255 | raise ex 256 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import fnmatch 3 | import logging 4 | import logging.config 5 | from collections import defaultdict 6 | import json 7 | import pytest 8 | 9 | import Cryptodome 10 | import yaml 11 | from mock import patch 12 | from pyfakefs.fake_filesystem_unittest import TestCase 13 | 14 | from ceph_salt.salt_utils import SaltClient 15 | 16 | 17 | logging.config.dictConfig({ 18 | 'version': 1, 19 | 'disable_existing_loggers': False, 20 | 'formatters': { 21 | 'standard': { 22 | 'format': '%(asctime)s [%(levelname)s] [%(name)s] %(message)s' 23 | }, 24 | }, 25 | 'handlers': { 26 | 'console': { 27 | 'level': 'DEBUG', 28 | 'class': 'logging.StreamHandler', 29 | 'formatter': 'standard' 30 | }, 31 | }, 32 | 'loggers': { 33 | '': { 34 | 'handlers': ['console'], 35 | 'level': 'DEBUG', 36 | 'propagate': True, 37 | } 38 | } 39 | }) 40 | 41 | 42 | logger = logging.getLogger(__name__) 43 | 44 | 45 | class ModuleUtil: 46 | @staticmethod 47 | def parse_module(module): 48 | return module.split('.', 1) 49 | 50 | 51 | class SaltEnv: 52 | minions = [] 53 | 54 | 55 | class SaltGrainsMock: 56 | def __init__(self): 57 | self.logger = logging.getLogger(SaltGrainsMock.__name__) 58 | self.grains = {} 59 | 60 | def setval(self, key, value): 61 | self.logger.info('setval %s, %s', key, value) 62 | self.grains[key] = value 63 | 64 | def get(self, key): 65 | self.logger.info('get %s', key) 66 | return self.grains.get(key, '') 67 | 68 | def delkey(self, key): 69 | self.logger.info('delkey %s', key) 70 | del self.grains[key] 71 | 72 | def enumerate_entries(self, _dict=None): 73 | if _dict is None: 74 | _dict = self.grains 75 | 76 | entries = [] 77 | for key, val in _dict.items(): 78 | if isinstance(val, dict): 79 | _entries = self.enumerate_entries(val) 80 | entries.extend(["{}:{}".format(key, e) for e in _entries]) 81 | elif isinstance(val, list): 82 | entries.extend(["{}:{}".format(key, e) for e in val]) 83 | elif isinstance(val, bool): 84 | entries.append('{}:{}'.format(key, val)) 85 | entries.append(key) 86 | else: 87 | entries.append('{}:{}'.format(key, val)) 88 | 89 | self.logger.debug("grains enumeration: %s -> %s", _dict, entries) 90 | return entries 91 | 92 | 93 | class TestMock: 94 | @staticmethod 95 | def ping(): 96 | return True 97 | 98 | @staticmethod 99 | def true(): 100 | return True 101 | 102 | 103 | class SaltUtilMock: 104 | sync_all_result = True 105 | 106 | @staticmethod 107 | def pillar_refresh(): 108 | return True 109 | 110 | @classmethod 111 | def sync_all(cls): 112 | return cls.sync_all_result 113 | 114 | @classmethod 115 | def running(cls): 116 | return False 117 | 118 | 119 | class StateMock: 120 | @staticmethod 121 | def sls_exists(state): 122 | return os.path.exists(os.path.join(SaltMockTestCase.states_fs_path(), state)) or \ 123 | os.path.exists(os.path.join(SaltMockTestCase.states_fs_path(), "{}.sls".format(state))) 124 | 125 | 126 | class ServiceMock: 127 | restart_result = True 128 | 129 | @classmethod 130 | def restart(cls, service): # pylint: disable=unused-argument 131 | return cls.restart_result 132 | 133 | 134 | class CephOrchMock: 135 | configured_result = True 136 | ceph_configured_result = True 137 | host_ls_result = [] 138 | 139 | @classmethod 140 | def configured(cls): 141 | return cls.configured_result 142 | 143 | @classmethod 144 | def ceph_configured(cls): 145 | return cls.ceph_configured_result 146 | 147 | @classmethod 148 | def host_ls(cls): 149 | return cls.host_ls_result 150 | 151 | 152 | class NetworkMock: 153 | subnets_result = [] 154 | 155 | @classmethod 156 | def subnets(cls): 157 | return cls.subnets_result 158 | 159 | 160 | class SaltLocalClientMock: 161 | 162 | def __init__(self): 163 | self.logger = logging.getLogger(SaltLocalClientMock.__name__) 164 | self.grains = defaultdict(SaltGrainsMock) 165 | 166 | def cmd(self, target, module, args=None, tgt_type=None, full_return=False): 167 | self.logger.info('cmd %s, %s, %s, tgt_type=%s, full_return=%s', 168 | target, module, args, tgt_type, full_return) 169 | 170 | if args is None: 171 | args = [] 172 | 173 | targets = [] 174 | if tgt_type == 'grain': 175 | for minion, grains in self.grains.items(): 176 | self.logger.info("grain filtering: %s <-> %s", grains.enumerate_entries(), target) 177 | if fnmatch.filter(grains.enumerate_entries(), target): 178 | targets.append(minion) 179 | else: 180 | targets.append(target) 181 | 182 | result = {} 183 | for tgt in targets: 184 | mod, func = ModuleUtil.parse_module(module) 185 | if mod == 'grains': 186 | ret = getattr(self.grains[tgt], func)(*args) 187 | elif mod == 'test': 188 | ret = getattr(TestMock, func)(*args) 189 | elif mod == 'saltutil': 190 | ret = getattr(SaltUtilMock, func)(*args) 191 | elif mod == 'state': 192 | ret = getattr(StateMock, func)(*args) 193 | elif mod == 'service': 194 | ret = getattr(ServiceMock, func)(*args) 195 | elif mod == 'ceph_orch': 196 | ret = getattr(CephOrchMock, func)(*args) 197 | elif mod == 'network': 198 | ret = getattr(NetworkMock, func)(*args) 199 | else: 200 | raise NotImplementedError() 201 | if full_return: 202 | result[tgt] = { 203 | 'ret': ret, 204 | 'retcode': 0 205 | } 206 | else: 207 | result[tgt] = ret 208 | 209 | self.logger.info("Grains: %s", self.grains) 210 | 211 | return result 212 | 213 | 214 | class MinionMock: 215 | @staticmethod 216 | def list(): 217 | return { 218 | 'minions': SaltEnv.minions, 219 | 'minions_denied': [], 220 | 'minions_pre': [], 221 | 'minions_rejected': [] 222 | } 223 | 224 | 225 | class SaltCallerMock: 226 | 227 | def cmd(self, fun, *args, **kwargs): 228 | mod, func = ModuleUtil.parse_module(fun) 229 | if mod == 'minion': 230 | return getattr(MinionMock, func)(*args, **kwargs) 231 | if mod == 'test': 232 | return getattr(TestMock, func)(*args, **kwargs) 233 | if mod == 'service': 234 | return getattr(ServiceMock, func)(*args) 235 | raise NotImplementedError() 236 | 237 | 238 | class SaltMasterMinionMock: 239 | pass 240 | 241 | 242 | class SaltMasterConfigMock: 243 | def __init__(self): 244 | self.opts = {'pillar_roots': {'base': ['/srv/pillar']}} 245 | 246 | def __call__(self, *args): 247 | logger.info("Getting salt options: %s", self.opts) 248 | return self.opts 249 | 250 | 251 | # pylint: disable=invalid-name 252 | class SaltMockTestCase(TestCase): 253 | def __init__(self, methodName='runTest'): 254 | super(SaltMockTestCase, self).__init__(methodName) 255 | self.capsys = None 256 | self.salt_env = SaltEnv 257 | 258 | @staticmethod 259 | def pillar_fs_path(): 260 | return '/srv/pillar' 261 | 262 | @staticmethod 263 | def states_fs_path(): 264 | return '/srv/salt' 265 | 266 | @staticmethod 267 | def pki_minions_fs_path(): 268 | return '/etc/salt/pki/master/minions' 269 | 270 | def setUp(self): 271 | super(SaltMockTestCase, self).setUp() 272 | self.setUpPyfakefs() 273 | # Make sure tests can access real files inside the source tree 274 | # (needed specifically for Cryptodome's dynamic library loading) 275 | self.fs.add_real_directory(os.path.dirname(Cryptodome.__file__)) 276 | self.local_fs = self.fs 277 | 278 | logger.info("Initializing Salt mocks") 279 | 280 | # pylint: disable=protected-access 281 | SaltClient._OPTS_ = None 282 | SaltClient._LOCAL_ = None 283 | SaltClient._CALLER_ = None 284 | SaltClient._MASTER_ = None 285 | 286 | self.caller_client = SaltCallerMock() 287 | self.local_client = SaltLocalClientMock() 288 | self.master_minion = SaltMasterMinionMock() 289 | self.master_config = SaltMasterConfigMock() 290 | 291 | patchers = [ 292 | patch('salt.config.master_config', new=self.master_config), 293 | patch('salt.client.Caller', return_value=self.caller_client), 294 | patch('salt.client.LocalClient', return_value=self.local_client), 295 | patch('salt.minion.MasterMinion', return_value=self.master_minion), 296 | patch('shutil.chown'), 297 | ] 298 | for patcher in patchers: 299 | patcher.start() 300 | self.addCleanup(patcher.stop) 301 | self.fs.create_dir(self.pillar_fs_path()) 302 | self.fs.create_dir(self.states_fs_path()) 303 | self.fs.create_dir(self.pki_minions_fs_path()) 304 | self.fs.create_file(os.path.join(self.pillar_fs_path(), 'ceph-salt.sls')) 305 | 306 | def tearDown(self): 307 | super(SaltMockTestCase, self).tearDown() 308 | self.fs.remove_object(os.path.join(self.pillar_fs_path(), 'ceph-salt.sls')) 309 | self.salt_env.minions = [] 310 | 311 | @pytest.fixture(autouse=True) 312 | def capsys(self, capsys): 313 | self.capsys = capsys 314 | 315 | def clearSysOut(self): 316 | self.capsys.readouterr() 317 | 318 | def assertInSysOut(self, text): 319 | out, _ = self.capsys.readouterr() 320 | self.assertIn(text, out) 321 | 322 | def assertJsonInSysOut(self, value): 323 | out, _ = self.capsys.readouterr() 324 | self.assertEqual(value, json.loads(out)) 325 | 326 | def assertGrains(self, target, key, value): 327 | self.assertIn(target, self.local_client.grains) 328 | target_grains = self.local_client.grains[target].grains 329 | self.assertIn(key, target_grains) 330 | self.assertEqual(target_grains[key], value) 331 | 332 | def assertNotInGrains(self, target, key): 333 | self.assertIn(target, self.local_client.grains) 334 | target_grains = self.local_client.grains[target].grains 335 | self.assertNotIn(key, target_grains) 336 | 337 | def assertYamlEqual(self, file_path, _dict): 338 | data = None 339 | with open(file_path, 'r') as file: 340 | data = yaml.full_load(file) 341 | self.assertDictEqual(data, _dict) 342 | -------------------------------------------------------------------------------- /tests/test_salt_event.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import unittest 4 | from typing import List 5 | 6 | import mock 7 | 8 | from ceph_salt.salt_event import SaltEventProcessor, EventListener, CephSaltEvent, \ 9 | SaltEvent, JobRetEvent 10 | 11 | 12 | # pylint: disable=unused-argument 13 | 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class SaltEventStream: 19 | _events = [] 20 | _handler = None 21 | 22 | @classmethod 23 | def push_event(cls, tag, data): 24 | logger.info("push event: %s, %s", tag, data) 25 | cls._events.append((tag, data)) 26 | 27 | @classmethod 28 | def flush_events(cls): 29 | if cls._handler: 30 | while cls._events: 31 | cls._handler(None) # pylint: disable=not-callable 32 | 33 | @classmethod 34 | def set_event_handler(cls, handler): 35 | logger.info("Registering handler: %s", handler) 36 | cls._handler = handler 37 | 38 | @classmethod 39 | def unpack(cls, *args, **kwargs): 40 | logger.info("Unpack event: %s", cls._events) 41 | tag, data = cls._events.pop(0) 42 | return tag, data 43 | 44 | 45 | class TestEventListener(EventListener): 46 | ceph_salt_events: List[CephSaltEvent] = [] 47 | begin_stage_events = [] 48 | warning_stage_events = [] 49 | end_stage_events = [] 50 | begin_step_events = [] 51 | end_step_events = [] 52 | minion_reboot_events = [] 53 | minion_start_events = [] 54 | state_apply_return_events = [] 55 | 56 | def handle_ceph_salt_event(self, event: CephSaltEvent): 57 | logger.info("received ceph-salt event: %s", event) 58 | self.ceph_salt_events.append(event) 59 | 60 | def handle_begin_stage(self, event: CephSaltEvent): 61 | self.begin_stage_events.append(event) 62 | 63 | def handle_warning_stage(self, event: CephSaltEvent): 64 | self.warning_stage_events.append(event) 65 | 66 | def handle_end_stage(self, event: CephSaltEvent): 67 | self.end_stage_events.append(event) 68 | 69 | def handle_begin_step(self, event: CephSaltEvent): 70 | self.begin_step_events.append(event) 71 | 72 | def handle_end_step(self, event: CephSaltEvent): 73 | self.end_step_events.append(event) 74 | 75 | def handle_minion_reboot(self, event: CephSaltEvent): 76 | self.minion_reboot_events.append(event) 77 | 78 | def handle_minion_start(self, event: SaltEvent): 79 | self.minion_start_events.append(event) 80 | 81 | def handle_state_apply_return(self, event: JobRetEvent): 82 | self.state_apply_return_events.append(event) 83 | 84 | 85 | class TestSaltEvent(unittest.TestCase): 86 | 87 | def __init__(self, methodName='runTest'): 88 | super(TestSaltEvent, self).__init__(methodName) 89 | self.processor = None 90 | 91 | def setUp(self): 92 | patchers = [ 93 | mock.patch('salt.utils.event.get_event', return_value=SaltEventStream), 94 | mock.patch('salt.config.client_config'), 95 | mock.patch("salt.utils.event.SaltEvent", new_callable=SaltEventStream), 96 | ] 97 | for patcher in patchers: 98 | patcher.start() 99 | self.addCleanup(patcher.stop) 100 | 101 | self.processor = SaltEventProcessor(['node1.test.com', 'node2.test.com']) 102 | self.processor.start() 103 | 104 | def tearDown(self): 105 | if self.processor.is_running(): 106 | self.processor.stop() 107 | self.processor = None 108 | 109 | def test_listener(self): 110 | listener = TestEventListener() 111 | self.processor.add_listener(listener) 112 | 113 | SaltEventStream.push_event('ceph-salt/stage/begin', { 114 | 'id': 'node1.test.com', 115 | 'cmd': '_minion_event', 116 | 'pretag': None, 117 | 'data': { 118 | 'desc': 'Doing stuff 1' 119 | }, 120 | 'tag': 'ceph-salt/stage/begin', 121 | '_stamp': '2020-01-17T15:19:54.719389' 122 | }) 123 | SaltEventStream.push_event('ceph-salt/step/begin', { 124 | 'id': 'node1.test.com', 125 | 'cmd': '_minion_event', 126 | 'pretag': None, 127 | 'data': { 128 | 'desc': 'Doing step 1' 129 | }, 130 | 'tag': 'ceph-salt/stage/begin', 131 | '_stamp': '2020-01-17T15:19:55.719389' 132 | }) 133 | SaltEventStream.push_event('ceph-salt/step/end', { 134 | 'id': 'node1.test.com', 135 | 'cmd': '_minion_event', 136 | 'pretag': None, 137 | 'data': { 138 | 'desc': 'Doing step 1' 139 | }, 140 | 'tag': 'ceph-salt/stage/begin', 141 | '_stamp': '2020-01-17T15:19:56.719389' 142 | }) 143 | SaltEventStream.push_event('ceph-salt/stage/end', { 144 | 'id': 'node1.test.com', 145 | 'cmd': '_minion_event', 146 | 'pretag': None, 147 | 'data': { 148 | 'desc': 'Doing stuff 1' 149 | }, 150 | 'tag': 'ceph-salt/stage/begin', 151 | '_stamp': '2020-01-17T15:19:57.719389' 152 | }) 153 | SaltEventStream.push_event('20200117161959615228', { 154 | 'minions': ['node1.test.com'], 155 | '_stamp': '2020-01-17T15:19:59.615651' 156 | }) 157 | SaltEventStream.push_event('ceph-salt/stage/begin', { 158 | 'id': 'node2.test.com', 159 | 'cmd': '_minion_event', 160 | 'pretag': None, 161 | 'data': { 162 | 'desc': 'Doing stuff 2' 163 | }, 164 | 'tag': 'ceph-salt/stage/begin', 165 | '_stamp': '2020-01-17T15:20:54.719389' 166 | }) 167 | SaltEventStream.push_event('ceph-salt/minion_reboot', { 168 | 'id': 'node2.test.com', 169 | 'data': { 170 | 'desc': 'Rebooting...' 171 | }, 172 | 'tag': 'ceph-salt/minion_reboot', 173 | '_stamp': '2020-01-17T15:20:54.719389' 174 | }) 175 | SaltEventStream.flush_events() 176 | 177 | tstamp1 = datetime.datetime.strptime('2020-01-17T15:19:54.719389', "%Y-%m-%dT%H:%M:%S.%f") 178 | tstamp2 = datetime.datetime.strptime('2020-01-17T15:19:55.719389', "%Y-%m-%dT%H:%M:%S.%f") 179 | tstamp3 = datetime.datetime.strptime('2020-01-17T15:19:56.719389', "%Y-%m-%dT%H:%M:%S.%f") 180 | tstamp4 = datetime.datetime.strptime('2020-01-17T15:19:57.719389', "%Y-%m-%dT%H:%M:%S.%f") 181 | tstamp5 = datetime.datetime.strptime('2020-01-17T15:20:54.719389', "%Y-%m-%dT%H:%M:%S.%f") 182 | tstamp6 = datetime.datetime.strptime('2020-01-17T15:20:54.719389', "%Y-%m-%dT%H:%M:%S.%f") 183 | 184 | self.assertEqual(len(listener.ceph_salt_events), 6) 185 | self.assertEqual(len(listener.begin_stage_events), 2) 186 | self.assertEqual(len(listener.end_stage_events), 1) 187 | self.assertEqual(len(listener.begin_step_events), 1) 188 | self.assertEqual(len(listener.end_step_events), 1) 189 | self.assertEqual(len(listener.minion_reboot_events), 1) 190 | 191 | self.assertEqual(listener.ceph_salt_events[0].minion, "node1.test.com") 192 | self.assertEqual(listener.ceph_salt_events[0].desc, "Doing stuff 1") 193 | self.assertEqual(listener.ceph_salt_events[0].tag, "ceph-salt/stage/begin") 194 | self.assertEqual(listener.ceph_salt_events[0].stamp, tstamp1) 195 | 196 | self.assertEqual(listener.ceph_salt_events[1].minion, "node1.test.com") 197 | self.assertEqual(listener.ceph_salt_events[1].desc, "Doing step 1") 198 | self.assertEqual(listener.ceph_salt_events[1].tag, "ceph-salt/step/begin") 199 | self.assertEqual(listener.ceph_salt_events[1].stamp, tstamp2) 200 | 201 | self.assertEqual(listener.ceph_salt_events[2].minion, "node1.test.com") 202 | self.assertEqual(listener.ceph_salt_events[2].desc, "Doing step 1") 203 | self.assertEqual(listener.ceph_salt_events[2].tag, "ceph-salt/step/end") 204 | self.assertEqual(listener.ceph_salt_events[2].stamp, tstamp3) 205 | 206 | self.assertEqual(listener.ceph_salt_events[3].minion, "node1.test.com") 207 | self.assertEqual(listener.ceph_salt_events[3].desc, "Doing stuff 1") 208 | self.assertEqual(listener.ceph_salt_events[3].tag, "ceph-salt/stage/end") 209 | self.assertEqual(listener.ceph_salt_events[3].stamp, tstamp4) 210 | 211 | self.assertEqual(listener.ceph_salt_events[4].minion, "node2.test.com") 212 | self.assertEqual(listener.ceph_salt_events[4].desc, "Doing stuff 2") 213 | self.assertEqual(listener.ceph_salt_events[4].tag, "ceph-salt/stage/begin") 214 | self.assertEqual(listener.ceph_salt_events[4].stamp, tstamp5) 215 | 216 | self.assertEqual(listener.ceph_salt_events[5].minion, "node2.test.com") 217 | self.assertEqual(listener.ceph_salt_events[5].desc, "Rebooting...") 218 | self.assertEqual(listener.ceph_salt_events[5].tag, "ceph-salt/minion_reboot") 219 | self.assertEqual(listener.ceph_salt_events[5].stamp, tstamp6) 220 | 221 | self.assertEqual(listener.ceph_salt_events[0], listener.begin_stage_events[0]) 222 | self.assertEqual(listener.ceph_salt_events[1], listener.begin_step_events[0]) 223 | self.assertEqual(listener.ceph_salt_events[2], listener.end_step_events[0]) 224 | self.assertEqual(listener.ceph_salt_events[3], listener.end_stage_events[0]) 225 | self.assertEqual(listener.ceph_salt_events[4], listener.begin_stage_events[1]) 226 | self.assertEqual(listener.ceph_salt_events[5], listener.minion_reboot_events[0]) 227 | 228 | def test_events_ignored(self): 229 | listener = TestEventListener() 230 | self.processor.add_listener(listener) 231 | 232 | SaltEventStream.push_event('minion_start', { 233 | 'id': 'node1.test.com', 234 | 'tag': 'minion_start', 235 | '_stamp': '2020-01-17T15:20:54.719389' 236 | }) 237 | SaltEventStream.push_event('minion_start', { 238 | 'id': 'node2.test.com', 239 | 'tag': 'minion_start', 240 | '_stamp': '2020-01-17T15:20:54.719389' 241 | }) 242 | SaltEventStream.push_event('minion_start', { 243 | 'id': 'node3.test.com', 244 | 'tag': 'minion_start', 245 | '_stamp': '2020-01-17T15:20:54.719389' 246 | }) 247 | SaltEventStream.flush_events() 248 | 249 | self.assertEqual(len(listener.minion_start_events), 2) 250 | 251 | def test_state_apply_return(self): 252 | listener = TestEventListener() 253 | self.processor.add_listener(listener) 254 | 255 | SaltEventStream.push_event('salt/job/20200215120107564789/ret/node1.test.com', { 256 | 'id': 'node1.test.com', 257 | 'tag': 'salt/job/20200215120107564789/ret/node1.test.com', 258 | '_stamp': '2020-01-17T15:20:54.719389', 259 | 'fun': 'state.apply', 260 | 'success': True 261 | }) 262 | SaltEventStream.flush_events() 263 | 264 | self.assertEqual(len(listener.state_apply_return_events), 1) 265 | -------------------------------------------------------------------------------- /ceph-salt-formula/salt/_states/ceph_orch.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import json 3 | import logging 4 | import time 5 | 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | def set_admin_host(name, if_grain=None, timeout=1800): 11 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 12 | if_grain_value = True 13 | if if_grain is not None: 14 | if_grain_value = __salt__['grains.get'](if_grain) 15 | if if_grain_value: 16 | __salt__['event.send']('ceph-salt/stage/begin', 17 | data={'desc': "Find an admin host"}) 18 | starttime = time.time() 19 | timelimit = starttime + timeout 20 | configured_admin_host = None 21 | while not configured_admin_host: 22 | is_timedout = time.time() > timelimit 23 | if is_timedout: 24 | ret['comment'] = 'Timeout value reached.' 25 | return ret 26 | time.sleep(15) 27 | bootstrap_minion = __pillar__['ceph-salt'].get('bootstrap_minion') 28 | if bootstrap_minion: 29 | failed = __salt__['ceph_salt.get_remote_grain'](bootstrap_minion, 'ceph-salt:execution:failed') 30 | if failed: 31 | ret['comment'] = 'Bootstrap minion failed.' 32 | return ret 33 | admin_hosts = __pillar__['ceph-salt']['minions']['admin'] 34 | for admin_host in admin_hosts: 35 | failed = __salt__['ceph_salt.get_remote_grain'](admin_host, 'ceph-salt:execution:failed') 36 | if failed: 37 | ret['comment'] = 'One or more admin minions failed.' 38 | return ret 39 | status_ret = __salt__['ceph_salt.ssh']( 40 | admin_host, 41 | "if [[ -f /etc/ceph/ceph.conf " 42 | "&& -f /etc/ceph/ceph.client.admin.keyring ]]; " 43 | "then timeout 60 sudo ceph -s; " 44 | "else (exit 1); fi") 45 | if status_ret['retcode'] == 0: 46 | configured_admin_host = admin_host 47 | break 48 | 49 | __salt__['event.send']('ceph-salt/stage/end', 50 | data={'desc': "Find an admin host"}) 51 | __salt__['grains.set']('ceph-salt:execution:admin_host', configured_admin_host) 52 | ret['result'] = True 53 | return ret 54 | 55 | def wait_until_ceph_orch_available(name, timeout=1800): 56 | """ 57 | Requires the following grains to be set: 58 | - ceph-salt:execution:admin_host 59 | """ 60 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 61 | starttime = time.time() 62 | timelimit = starttime + timeout 63 | while True: 64 | is_timedout = time.time() > timelimit 65 | if is_timedout: 66 | ret['comment'] = 'Timeout value reached.' 67 | return ret 68 | time.sleep(15) 69 | admin_host = __salt__['grains.get']('ceph-salt:execution:admin_host') 70 | status_ret = __salt__['ceph_salt.ssh']( 71 | admin_host, 72 | "if [[ -f /etc/ceph/ceph.conf " 73 | "&& -f /etc/ceph/ceph.client.admin.keyring ]]; " 74 | "then timeout 60 sudo ceph orch status --format=json; " 75 | "else (exit 1); fi") 76 | if status_ret['retcode'] == 0: 77 | status = json.loads(status_ret['stdout']) 78 | if status.get('available'): 79 | break 80 | ret['result'] = True 81 | return ret 82 | 83 | def add_host(name, host, ipaddr, is_admin=False): 84 | """ 85 | Requires the following grains to be set: 86 | - ceph-salt:execution:admin_host 87 | """ 88 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 89 | admin_host = __salt__['grains.get']('ceph-salt:execution:admin_host') 90 | cmd_ret = __salt__['ceph_salt.ssh']( 91 | admin_host, 92 | "sudo ceph orch host add {} {}{}".format( 93 | host, 94 | ipaddr, 95 | ' _admin' if is_admin else '', 96 | ), 97 | attempts=10) 98 | if cmd_ret['retcode'] == 0: 99 | ret['result'] = True 100 | else: 101 | ret['comment'] = cmd_ret.get('stderr') 102 | return ret 103 | 104 | def add_host_label(name, host, label): 105 | """ 106 | Requires the following grains to be set: 107 | - ceph-salt:execution:admin_host 108 | """ 109 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 110 | admin_host = __salt__['grains.get']('ceph-salt:execution:admin_host') 111 | cmd_ret = __salt__['ceph_salt.ssh']( 112 | admin_host, 113 | "sudo ceph orch host label add {} {}".format( 114 | host, 115 | label, 116 | ), 117 | attempts=10) 118 | if cmd_ret['retcode'] == 0: 119 | ret['result'] = True 120 | else: 121 | ret['comment'] = cmd_ret.get('stderr') 122 | return ret 123 | 124 | def rm_clusters(name): 125 | """ 126 | Requires the following pillar to be set: 127 | - ceph-salt:execution:fsid 128 | """ 129 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 130 | fsid = __salt__['pillar.get']('ceph-salt:execution:fsid') 131 | __salt__['ceph_salt.begin_stage']("Remove cluster {}".format(fsid)) 132 | cmd_ret = __salt__['cmd.run_all']("cephadm rm-cluster --fsid {} " 133 | "--force".format(fsid)) 134 | if cmd_ret['retcode'] == 0: 135 | __salt__['ceph_salt.end_stage']("Remove cluster {}".format(fsid)) 136 | ret['result'] = True 137 | else: 138 | ret['comment'] = cmd_ret.get('stderr') 139 | return ret 140 | 141 | 142 | def copy_ceph_conf_and_keyring_from_admin(name): 143 | """ 144 | Requires the following grains to be set: 145 | - ceph-salt:execution:admin_host 146 | """ 147 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 148 | admin_host = __salt__['grains.get']('ceph-salt:execution:admin_host') 149 | cmd_ret = __salt__['ceph_salt.sudo_rsync']( 150 | "cephadm@{}:/etc/ceph/{{ceph.conf,ceph.client.admin.keyring}}".format(admin_host), 151 | "/etc/ceph/", 152 | False) 153 | if cmd_ret['retcode'] == 0: 154 | ret['result'] = True 155 | else: 156 | ret['comment'] = cmd_ret.get('stderr') 157 | return ret 158 | 159 | 160 | def copy_ceph_conf_and_keyring_to_any_admin(name): 161 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 162 | admin_host = __pillar__['ceph-salt']['minions']['admin'][0] 163 | cmd_ret = __salt__['ceph_salt.sudo_rsync']( 164 | "/tmp/ceph.client.admin.keyring", 165 | "cephadm@{}:/etc/ceph/ceph.client.admin.keyring".format(admin_host), 166 | True) 167 | if cmd_ret['retcode'] != 0: 168 | ret['comment'] = cmd_ret.get('stderr') 169 | return ret 170 | cmd_ret = __salt__['ceph_salt.sudo_rsync']( 171 | "/etc/ceph/ceph.conf", 172 | "cephadm@{}:/etc/ceph/".format(admin_host), 173 | True) 174 | if cmd_ret['retcode'] != 0: 175 | ret['comment'] = cmd_ret.get('stderr') 176 | return ret 177 | ret['result'] = True 178 | return ret 179 | 180 | 181 | def wait_for_ceph_orch_host_ok_to_stop(name, if_grain, timeout=36000): 182 | """ 183 | Requires the following grains to be set: 184 | - ceph-salt:execution:admin_host 185 | """ 186 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 187 | if_grain_value = __salt__['grains.get'](if_grain) 188 | if if_grain_value: 189 | host = __grains__['host'] 190 | __salt__['event.send']('ceph-salt/stage/begin', 191 | data={'desc': "Wait for 'ceph orch host ok-to-stop {}'".format(host)}) 192 | ok_to_stop = False 193 | starttime = time.time() 194 | timelimit = starttime + timeout 195 | while not ok_to_stop: 196 | is_timedout = time.time() > timelimit 197 | if is_timedout: 198 | ret['comment'] = 'Timeout value reached.' 199 | return ret 200 | admin_host = __salt__['grains.get']('ceph-salt:execution:admin_host') 201 | cmd_ret = __salt__['ceph_salt.ssh']( 202 | admin_host, 203 | "sudo ceph orch host ok-to-stop {}".format(host)) 204 | ok_to_stop = cmd_ret['retcode'] == 0 205 | if not ok_to_stop: 206 | logger.info("Waiting for 'ceph_orch.host_ok_to_stop'") 207 | time.sleep(15) 208 | __salt__['event.send']('ceph-salt/stage/end', 209 | data={'desc': "Wait for 'ceph orch host ok-to-stop {}'".format(host)}) 210 | ret['result'] = True 211 | return ret 212 | 213 | def stop_service_by_daemon_name(ceph_orch_ps_ret, ret): 214 | json_obj = json.loads(ceph_orch_ps_ret) 215 | for elem in json_obj: 216 | cmd_ret = __salt__['cmd.run_all']( 217 | "ceph orch daemon stop {}".format(elem['daemon_name'])) 218 | if cmd_ret['retcode'] != 0: 219 | ret['result'] = False 220 | ret['comment'] = cmd_ret.get('stderr') 221 | 222 | def stop_service(name, service): 223 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': True} 224 | ceph_orch_ps_ret = __salt__['cmd.run_all']( 225 | "ceph orch ps --daemon_type {} --format json".format(service)) 226 | stop_service_by_daemon_name(ceph_orch_ps_ret['stdout'], ret) 227 | return ret 228 | 229 | def wait_until_service_stopped(name, service, timeout=1800): 230 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 231 | service_stopped = False 232 | starttime = time.time() 233 | timelimit = starttime + timeout 234 | while not service_stopped: 235 | is_timedout = time.time() > timelimit 236 | if is_timedout: 237 | ret['comment'] = 'Timeout value reached.' 238 | return ret 239 | cmd_ret = __salt__['cmd.run_all']( 240 | "ceph orch ls --service-type {} --format json".format(service)) 241 | if cmd_ret['retcode'] != 0: 242 | ret['comment'] = cmd_ret.get('stderr') 243 | return ret 244 | try: 245 | status = json.loads(cmd_ret['stdout'])[0]['status'] 246 | service_stopped = status['running'] == 0 247 | except json.decoder.JSONDecodeError as exc: 248 | if 'No services reported' in cmd_ret['stdout']: 249 | service_stopped = True 250 | else: 251 | raise exc 252 | ret['result'] = True 253 | return ret 254 | 255 | 256 | def stop_ceph_fsid(name): 257 | """ 258 | Requires the following pillar to be set: 259 | - ceph-salt:execution:fsid 260 | """ 261 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 262 | fsid = __salt__['pillar.get']('ceph-salt:execution:fsid') 263 | service = 'ceph-{}.target'.format(fsid) 264 | __salt__['ceph_salt.begin_stage']("Stop '{}'".format(service)) 265 | cmd_ret = __salt__['cmd.run_all']("systemctl stop {} ".format(service)) 266 | if cmd_ret['retcode'] == 0: 267 | __salt__['ceph_salt.end_stage']("Stop '{}'".format(service)) 268 | ret['result'] = True 269 | else: 270 | ret['comment'] = cmd_ret.get('stderr') 271 | return ret 272 | 273 | 274 | def set_osd_flag(name, flag): 275 | ret = {'name': name, 'changes': {}, 'comment': '', 'result': False} 276 | cmd_ret = __salt__['cmd.run_all']( 277 | "ceph osd set {}".format(flag)) 278 | if cmd_ret['retcode'] == 0: 279 | ret['result'] = True 280 | else: 281 | ret['comment'] = cmd_ret.get('stderr') 282 | return ret 283 | -------------------------------------------------------------------------------- /tests/test_execute.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import threading 3 | import time 4 | import logging 5 | import os 6 | 7 | import mock 8 | import pytest 9 | 10 | from ceph_salt.execute import CephSaltController, TerminalRenderer, CephSaltModel, Event, \ 11 | CursesRenderer, CephSaltExecutor 12 | from ceph_salt.exceptions import MinionDoesNotExistInConfiguration 13 | from ceph_salt.salt_utils import GrainsManager 14 | from ceph_salt.salt_event import CephSaltEvent 15 | from ceph_salt.salt_utils import PillarManager 16 | 17 | from . import SaltMockTestCase, ServiceMock, SaltUtilMock, CephOrchMock 18 | 19 | 20 | # pylint: disable=unused-argument 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | def _event(tag, minion_id, desc, sec): 25 | return CephSaltEvent({ 26 | 'tag': 'ceph-salt/{}'.format(tag), 27 | 'data': { 28 | 'id': minion_id, 29 | 'cmd': '_minion_event', 30 | 'pretag': None, 31 | 'data': { 32 | 'desc': desc 33 | }, 34 | 'tag': 'ceph-salt/stage/begin', 35 | '_stamp': '2020-01-17T15:19:{}.719389'.format(sec) 36 | } 37 | }) 38 | 39 | 40 | def begin_stage(minion_id, desc, sec): 41 | return _event('stage/begin', minion_id, desc, sec) 42 | 43 | 44 | def end_stage(minion_id, desc, sec): 45 | return _event('stage/end', minion_id, desc, sec) 46 | 47 | 48 | def begin_step(minion_id, desc, sec): 49 | return _event('step/begin', minion_id, desc, sec) 50 | 51 | 52 | def end_step(minion_id, desc, sec): 53 | return _event('step/end', minion_id, desc, sec) 54 | 55 | 56 | def failure(): 57 | return { 58 | "state": "file_|-/etc/chrony.conf_|-/etc/chrony.conf_|-managed", 59 | "__id__": "/etc/chrony.conf", 60 | "__run_num__": 21, 61 | "__sls__": "ceph-salt.time", 62 | "changes": {}, 63 | "comment": "Unable to manage file: Jinja variable 'dict object' has no attribute " 64 | "'external_time_servers'", 65 | "duration": 49.631, 66 | "name": "/etc/chrony.conf", 67 | "pchanges": {}, 68 | "result": False, 69 | "start_time": "16:46:46.337877" 70 | } 71 | 72 | 73 | class ApplyTest(SaltMockTestCase): 74 | def setUp(self): 75 | super(ApplyTest, self).setUp() 76 | self.salt_env.minions = ['node1.ceph.com', 'node2.ceph.com'] 77 | GrainsManager.set_grain('node1.ceph.com', 'host', 'node1') 78 | GrainsManager.set_grain('node2.ceph.com', 'host', 'node2') 79 | GrainsManager.set_grain('node1.ceph.com', 'fqdn_ip4', ['10.20.39.201']) 80 | GrainsManager.set_grain('node2.ceph.com', 'fqdn_ip4', ['10.20.39.202']) 81 | GrainsManager.set_grain('node1.ceph.com', 'ceph-salt', {'member': True, 82 | 'roles': ['mon'], 83 | 'execution': {}}) 84 | GrainsManager.set_grain('node2.ceph.com', 'ceph-salt', {'member': True, 85 | 'roles': ['mgr'], 86 | 'execution': {}}) 87 | 88 | def tearDown(self): 89 | super(ApplyTest, self).tearDown() 90 | PillarManager.reload() 91 | 92 | def test_minion_does_not_exist(self): 93 | with pytest.raises(MinionDoesNotExistInConfiguration): 94 | CephSaltModel('node3.ceph.com', 'ceph-salt', {}) 95 | 96 | def test_controller_with_terminal_renderer(self): 97 | model = CephSaltModel(None, 'ceph-salt', {}) 98 | renderer = TerminalRenderer(model) 99 | controller = CephSaltController(model, renderer) 100 | 101 | controller.begin() 102 | time.sleep(0.2) 103 | controller.handle_begin_stage(begin_stage('node1.ceph.com', 'Stage 1', 49)) 104 | time.sleep(0.2) 105 | controller.handle_begin_stage(begin_stage('node2.ceph.com', 'Stage 1', 50)) 106 | time.sleep(0.2) 107 | controller.handle_begin_step(begin_step('node1.ceph.com', 'Step 1', 51)) 108 | time.sleep(0.2) 109 | controller.handle_end_step(end_step('node1.ceph.com', 'Step 1', 52)) 110 | time.sleep(0.2) 111 | controller.handle_begin_step(begin_step('node2.ceph.com', 'Step 2', 53)) 112 | time.sleep(0.2) 113 | controller.handle_end_step(end_step('node2.ceph.com', 'Step 2', 54)) 114 | time.sleep(0.2) 115 | controller.handle_end_stage(end_stage('node2.ceph.com', 'Stage 1', 55)) 116 | time.sleep(0.2) 117 | controller.handle_end_stage(end_stage('node1.ceph.com', 'Stage 1', 56)) 118 | 119 | tstamp1 = datetime.datetime.strptime('2020-01-17T15:19:58.819390', "%Y-%m-%dT%H:%M:%S.%f") 120 | tstamp2 = datetime.datetime.strptime('2020-01-17T15:19:59.819390', "%Y-%m-%dT%H:%M:%S.%f") 121 | controller.minion_finished('node1.ceph.com', tstamp2, False) 122 | controller.minion_finished('node2.ceph.com', tstamp1, False) 123 | 124 | time.sleep(0.2) 125 | controller.minion_failure('node2.ceph.com', Event('begin_step', 'Step 2', 126 | Event('begin_stage', 'Stage 1')), 127 | failure()) 128 | time.sleep(0.2) 129 | controller.minion_failure('node1.ceph.com', Event('end_step', 'Step 1', 130 | Event('begin_stage', 'Stage 1')), 131 | failure()) 132 | 133 | time.sleep(0.2) 134 | controller.end() 135 | 136 | self.assertEqual(len(model.minions_list()), 2) 137 | self.assertIsNotNone(model.begin_time) 138 | self.assertTrue(model.finished()) 139 | self.assertEqual(model.minions_finished(), 2) 140 | self.assertEqual(model.minions_total(), 2) 141 | self.assertEqual(model.minions_failed(), 2) 142 | self.assertEqual(model.minions_succeeded(), 0) 143 | 144 | node1 = model.get_minion('node1.ceph.com') 145 | self.assertTrue(node1.finished()) 146 | self.assertEqual(len(node1.stages), 1) 147 | self.assertFalse(node1.success) 148 | stage = node1.last_stage 149 | self.assertEqual(stage.desc, 'Stage 1') 150 | self.assertEqual(len(stage.steps), 2) 151 | self.assertFalse(stage.success) 152 | steps = list(stage.steps.values()) 153 | self.assertEqual(steps[0].desc, 'Step 1') 154 | self.assertTrue(steps[0].finished()) 155 | self.assertTrue(steps[0].success) 156 | self.assertIsInstance(steps[1], dict) 157 | self.assertEqual(steps[1]['state'], 'file_|-/etc/chrony.conf_|-/etc/chrony.conf_|-managed') 158 | 159 | node2 = model.get_minion('node2.ceph.com') 160 | self.assertTrue(node2.finished()) 161 | self.assertEqual(len(node2.stages), 1) 162 | self.assertFalse(node2.success) 163 | stage = node2.last_stage 164 | self.assertEqual(stage.desc, 'Stage 1') 165 | self.assertEqual(len(stage.steps), 1) 166 | self.assertFalse(stage.success) 167 | step = stage.last_step 168 | self.assertEqual(step.desc, 'Step 2') 169 | self.assertTrue(step.finished()) 170 | self.assertFalse(step.success) 171 | self.assertIsNotNone(step.failure) 172 | self.assertEqual(step.failure['state'], 173 | 'file_|-/etc/chrony.conf_|-/etc/chrony.conf_|-managed') 174 | 175 | @mock.patch('curses.color_pair') 176 | @mock.patch('curses.newwin') 177 | @mock.patch('curses.endwin') 178 | @mock.patch('curses.curs_set') 179 | @mock.patch('curses.nocbreak') 180 | @mock.patch('curses.cbreak') 181 | @mock.patch('curses.noecho') 182 | @mock.patch('curses.echo') 183 | @mock.patch('curses.init_pair') 184 | @mock.patch('curses.use_default_colors') 185 | @mock.patch('curses.start_color') 186 | @mock.patch('curses.newpad') 187 | @mock.patch('curses.initscr') 188 | def test_controller_with_curses_renderer(self, initscr, newpad, *args): 189 | stdscr = mock.MagicMock() 190 | initscr.return_value = stdscr 191 | stdscr.getmaxyx = mock.MagicMock(return_value=(10, 80)) 192 | 193 | class FakeGetCh: 194 | def __init__(self): 195 | self.next_char = None 196 | 197 | def __call__(self): 198 | time.sleep(0.2) 199 | cha = -1 if self.next_char is None else self.next_char 200 | self.next_char = None 201 | return cha 202 | 203 | fake_getch = FakeGetCh() 204 | stdscr.getch = mock.MagicMock(side_effect=fake_getch) 205 | 206 | body = mock.MagicMock() 207 | newpad.return_value = body 208 | 209 | class Addstr: 210 | def __init__(self): 211 | self.current_row = 0 212 | 213 | def __call__(self, row, *args): 214 | self.current_row = row 215 | 216 | def getyx(self): 217 | logger.info("return body current pos: %s, 80", self.current_row) 218 | return self.current_row, 80 219 | 220 | addstr = Addstr() 221 | body.addstr = mock.MagicMock(side_effect=addstr) 222 | body.getyx = mock.MagicMock(side_effect=addstr.getyx) 223 | 224 | model = CephSaltModel(None, 'ceph-salt', {}) 225 | renderer = CursesRenderer(model) 226 | controller = CephSaltController(model, renderer) 227 | 228 | class ExecutionThread(threading.Thread): 229 | def run(self): 230 | while not renderer.running: 231 | time.sleep(0.2) 232 | logger.info("starting injecting salt events in controller") 233 | 234 | controller.begin() 235 | time.sleep(0.3) 236 | controller.handle_begin_stage(begin_stage('node1.ceph.com', 'Stage 1', 49)) 237 | time.sleep(0.3) 238 | controller.handle_begin_stage(begin_stage('node2.ceph.com', 'Stage 1', 50)) 239 | time.sleep(0.3) 240 | controller.handle_begin_step(begin_step('node1.ceph.com', 'Step 1', 51)) 241 | time.sleep(0.3) 242 | controller.handle_end_step(end_step('node1.ceph.com', 'Step 1', 52)) 243 | time.sleep(0.3) 244 | controller.handle_begin_step(begin_step('node2.ceph.com', 'Step 2', 53)) 245 | fake_getch.next_char = ord('c') 246 | time.sleep(0.3) 247 | controller.handle_end_step(end_step('node2.ceph.com', 'Step 2', 54)) 248 | fake_getch.next_char = ord('j') 249 | time.sleep(0.3) 250 | fake_getch.next_char = ord('j') 251 | controller.handle_end_stage(end_stage('node2.ceph.com', 'Stage 1', 55)) 252 | time.sleep(0.3) 253 | fake_getch.next_char = ord('j') 254 | controller.handle_end_stage(end_stage('node1.ceph.com', 'Stage 1', 56)) 255 | 256 | tstamp1 = datetime.datetime.strptime('2020-01-17T15:19:58.819390', 257 | "%Y-%m-%dT%H:%M:%S.%f") 258 | tstamp2 = datetime.datetime.strptime('2020-01-17T15:19:59.819390', 259 | "%Y-%m-%dT%H:%M:%S.%f") 260 | controller.minion_finished('node1.ceph.com', tstamp2, False) 261 | controller.minion_finished('node2.ceph.com', tstamp1, False) 262 | 263 | time.sleep(0.3) 264 | controller.minion_failure( 265 | 'node2.ceph.com', 266 | Event('begin_step', 'Step 2', Event('begin_stage', 'Stage 1')), failure()) 267 | time.sleep(0.3) 268 | controller.minion_failure( 269 | 'node1.ceph.com', 270 | Event('end_step', 'Step 1', Event('begin_stage', 'Stage 1')), failure()) 271 | 272 | time.sleep(0.3) 273 | controller.end() 274 | fake_getch.next_char = ord('q') 275 | 276 | exec_thread = ExecutionThread() 277 | exec_thread.setDaemon(True) 278 | exec_thread.start() 279 | 280 | renderer.run() 281 | 282 | self.assertEqual(len(model.minions_list()), 2) 283 | self.assertIsNotNone(model.begin_time) 284 | self.assertTrue(model.finished()) 285 | self.assertEqual(model.minions_finished(), 2) 286 | self.assertEqual(model.minions_total(), 2) 287 | self.assertEqual(model.minions_failed(), 2) 288 | self.assertEqual(model.minions_succeeded(), 0) 289 | 290 | node1 = model.get_minion('node1.ceph.com') 291 | self.assertTrue(node1.finished()) 292 | self.assertEqual(len(node1.stages), 1) 293 | self.assertFalse(node1.success) 294 | stage = node1.last_stage 295 | self.assertEqual(stage.desc, 'Stage 1') 296 | self.assertEqual(len(stage.steps), 2) 297 | self.assertFalse(stage.success) 298 | steps = list(stage.steps.values()) 299 | self.assertEqual(steps[0].desc, 'Step 1') 300 | self.assertTrue(steps[0].finished()) 301 | self.assertTrue(steps[0].success) 302 | self.assertIsInstance(steps[1], dict) 303 | self.assertEqual(steps[1]['state'], 'file_|-/etc/chrony.conf_|-/etc/chrony.conf_|-managed') 304 | 305 | node2 = model.get_minion('node2.ceph.com') 306 | self.assertTrue(node2.finished()) 307 | self.assertEqual(len(node2.stages), 1) 308 | self.assertFalse(node2.success) 309 | stage = node2.last_stage 310 | self.assertEqual(stage.desc, 'Stage 1') 311 | self.assertEqual(len(stage.steps), 1) 312 | self.assertFalse(stage.success) 313 | step = stage.last_step 314 | self.assertEqual(step.desc, 'Step 2') 315 | self.assertTrue(step.finished()) 316 | self.assertFalse(step.success) 317 | self.assertIsNotNone(step.failure) 318 | self.assertEqual(step.failure['state'], 319 | 'file_|-/etc/chrony.conf_|-/etc/chrony.conf_|-managed') 320 | 321 | def _prompt_proceed(self, msg, default): 322 | pass 323 | 324 | def test_check_formula_ok(self): 325 | self.fs.create_file(os.path.join(self.states_fs_path(), 'ceph-salt.sls')) 326 | self.assertEqual(CephSaltExecutor.check_formula('ceph-salt', self._prompt_proceed), 0) 327 | self.fs.remove_object(os.path.join(self.states_fs_path(), 'ceph-salt.sls')) 328 | 329 | def test_check_formula_exists1(self): 330 | ServiceMock.restart_result = False 331 | self.assertEqual(CephSaltExecutor.check_formula('ceph-salt', self._prompt_proceed), 6) 332 | ServiceMock.restart_result = True 333 | 334 | def test_check_formula_exists2(self): 335 | self.assertEqual(CephSaltExecutor.check_formula('ceph-salt', self._prompt_proceed), 7) 336 | 337 | def test_check_sync_all(self): 338 | SaltUtilMock.sync_all_result = False 339 | self.fs.create_file(os.path.join(self.states_fs_path(), 'ceph-salt.sls')) 340 | self.assertEqual(CephSaltExecutor.check_sync_all(), 2) 341 | SaltUtilMock.sync_all_result = True 342 | self.fs.remove_object(os.path.join(self.states_fs_path(), 'ceph-salt.sls')) 343 | 344 | def test_check_cluster_day1_with_minion(self): 345 | self.fs.create_file(os.path.join(self.states_fs_path(), 'ceph-salt.sls')) 346 | self.assertEqual(CephSaltExecutor.check_cluster('ceph-salt', 'node1.ceph.com', []), 9) 347 | self.fs.remove_object(os.path.join(self.states_fs_path(), 'ceph-salt.sls')) 348 | 349 | def test_check_minion_not_found(self): 350 | self.fs.create_file(os.path.join(self.states_fs_path(), 'ceph-salt.sls')) 351 | host_ls_result = [{'hostname': 'node1'}] 352 | CephOrchMock.host_ls_result = host_ls_result 353 | self.assertEqual(CephSaltExecutor.check_cluster('ceph-salt', 354 | 'node9.ceph.com', host_ls_result), 10) 355 | CephOrchMock.host_ls_result = [] 356 | self.fs.remove_object(os.path.join(self.states_fs_path(), 'ceph-salt.sls')) 357 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ## [16.2.5] - 2023-09-04 11 | ### Fixed 12 | - Remove deprecated 'transport' kwarg 13 | 14 | ## [16.2.4] - 2022-12-19 15 | ### Fixed 16 | - Use daemon name rather than service type when stopping services 17 | - Run `saltutil.sync_runners` in ceph-salt-formula %post script (#486) 18 | - Don't explicitly install ceph-common (rely on ceph-base dependency) 19 | - Support monochrome terminals in `ceph-salt apply` 20 | 21 | ## [16.2.3] - 2022-04-14 22 | ### Fixed 23 | - Fix ceph-salt update when run prior to cluster deployment (#482) 24 | 25 | ## [16.2.2] - 2022-03-24 26 | ### Added 27 | - Add OS and Ceph version info to `ceph-salt status` output (#478) 28 | - add the orchestrator `_admin` host label during `ceph-salt update` (#476) 29 | 30 | ### Fixed 31 | - config the ssh key after package install/upgrade (#474) 32 | 33 | ## [16.2.1] - 2022-02-24 34 | ### Fixed 35 | - use ipaddress module to determine loopback interfaces (#472) 36 | 37 | ## [16.2.0] - 2021-12-01 38 | ### Added 39 | - Support deploying Ceph Pacific 40 | - ceph-salt-formula: Add hosts with admin label (#457) 41 | - ceph-salt-formula: give explicit IP addr when adding host (#465) 42 | 43 | ### Fixed 44 | - Tox: Fix issues when running tox on Python 3.8 (#457) 45 | - Remove Travis CI (#459) 46 | - .github/workflows: add tox and more python versions (#458) 47 | 48 | ## [15.2.16] - 2021-11-26 49 | ### Added 50 | - _modules/ceph_salt: log SSH commands (#464) 51 | 52 | ### Fixed 53 | - Move ceph-salt-registry-json creation to container.sls (#467) 54 | - Use `cephadm registry-login --registry-json` (#467) 55 | - Rely on cephadm package for cephadm user creation (#461) 56 | - README.md: Fix broken cephadm link (#455) 57 | - tests: add source dir as real directory to fake fs (#462) 58 | - .github/workflows/linting: use ceph_salt instead old name (#458) 59 | 60 | ## [15.2.15] - 2021-02-09 61 | ### Added 62 | - Add support for Salt 3002 (#449) 63 | - Add man page (#440) 64 | - Support multiple time servers (#439) 65 | - Add ceph-osd sysctl settings (#438) 66 | 67 | ### Fixed 68 | - config_shell: remove 0-length prefs.bin before init and handle init failure (#444) 69 | - Pin version of ceph-salt-formula RPM (#442) 70 | - Ensure mgr/cephadm/container_init is set to true (#437) 71 | 72 | ## [15.2.14] - 2020-10-14 73 | ### Added 74 | - Do not create ceph-salt pillar file during installation (#433) 75 | - Pillar targeting by minion id instead of grain (#432) 76 | - Optimize 'ceph-salt config' minion add (#430) 77 | - Inform user when default values are being populated (#431) 78 | - Fix salt warning on 'ceph-salt status' (#429) 79 | - Optimize 'ceph-salt config' load (#428) 80 | - Optimize 'ceph-salt status' (#428) 81 | 82 | ## [15.2.13] - 2020-10-08 83 | ### Added 84 | - Use '--container-init' option on 'cephadm bootstrap' (#396) 85 | - Support FQDN environments (#422) 86 | ### Fixed 87 | - Spell "Resetting" correctly (#423) 88 | - Add "Conflicts: deepsea-cli" to spec file (#421) 89 | - Expand the userpath during ssh key import (#425) 90 | - Reduce ceph-salt pillar file permissions (#415) 91 | 92 | ## [15.2.12] - 2020-09-28 93 | ### Added 94 | - Convert tags to repo_digest (#397) 95 | - Add 'ceph-salt stop' command (#404) 96 | - Validate duplicated registries (#403) 97 | - Validate '/cephadm_bootstrap/mon_ip' (#401) 98 | - Retry SSH 'host add' execution on connection closed (#398) 99 | ### Changed 100 | - Change cephadm bootstrap output redirect file (#395) 101 | ### Fixed 102 | - Handle unresponsive minions properly (#411) 103 | - Display 'stderr' on stage/step failure (#408) 104 | - Restart 'chronyd' service to apply chrony config (#407) 105 | - Install 'sudo' before 'sudoers' configuration (#400) 106 | - Fix salt warn when 'ceph-salt config' is executed for the first time (#399) 107 | 108 | ## [15.2.11] - 2020-09-11 109 | ### Added 110 | - Ask user confirmation before restarting 'salt-master' service (#392) 111 | - Execute salt sync-all on 'ceph-salt' config and status commands (#391) 112 | - Support bootstrap minion without admin role (#383) 113 | - Ignore 'cephbootstrap' salt state when ceph cluster already running (#385) 114 | - Add tuned 'latency' and 'throughput' roles (#361, #364, #387) 115 | - Improve SSH keys stage description (#376) 116 | - Remove 'ceph-salt:execution:provisioned' grain (#374) 117 | - Sanity-check time sync services when /time_server is disabled (#367) 118 | - Always use SSH 'cephadm' user (#363) 119 | - Use salt module for SSH executions (#363) 120 | - Set 'UserKnownHostsFile' and 'ConnectTimeout' on SSH connections (#363) 121 | - Verify whether minion nodes can resolve hostnames (#356) 122 | - Install ceph-salt ssh keys on admin minions (#257) 123 | - Add 'test.ping' sanity check (#354) 124 | ### Changed 125 | - Move ceph image path config to bootstrap section (#386) 126 | - Only pull ceph image on bootstrap minion (#353) 127 | ### Fixed 128 | - Install 'sudo' package (#384) 129 | - Fix reboot in parallel with cluster running (#380) 130 | - Rename '/etc/sudoers.d' file to avoid collision with cephadm (#363) 131 | - Bootstrap minion don't need to wait for other minions (#351) 132 | 133 | ## [15.2.10] - 2020-08-31 134 | ### Added 135 | - Add 'ceph-salt reboot [--force] [minion_id]' command (#325) 136 | - Write INFO message to log when stage/step begins/ends (#344) 137 | - Test on Python 3.8 instead of 3.7 (#338) 138 | - Add 'network' runner (#329) 139 | - Check for running jobs when failing to re-apply formula (#332) 140 | ### Removed 141 | - Remove '/system_update' config option (#345) 142 | ### Fixed 143 | - More meaningful SSH key comment (#337) 144 | 145 | ## [15.2.9] - 2020-08-10 146 | ### Added 147 | - Add 'ceph-salt update [--reboot] [minion_id]' command (#303) 148 | - Enable 'ceph.conf' management by cephadm (#291) 149 | - Suggest 'secure=false' on custom registries configuration help (#323) 150 | - Make use of cephadm for registry authentication (#295) 151 | 152 | ## [15.2.8] - 2020-08-10 153 | ### Added 154 | - Change default SSH user from 'root' to 'ceph-salt' (#319) 155 | - Make bootstrap minion optional (#315) 156 | - Purge ceph cluster (#306) 157 | - Install 'ceph-base' on admin minions (#305) 158 | - Declare RPM dependencies in spec file (#302) 159 | ### Fixed 160 | - 'cephadm' MGR module should only be enabled after cluster bootstrapped (#322) 161 | - Support SSH users that are configured by packages (#318) 162 | - aa-teardown can fail if apparmor is disabled on the boot command line (#316) 163 | - Configure 'qualified-search-registries = ["docker.io"]' (#321) 164 | - SSH pub and priv key should be set via 'cephadm' (#312) 165 | - Only create a single MON and MGR during bootstrap (#308) 166 | - Handle value errors on command line (#313) 167 | - Omit chrony.conf useless and counterproductive options (#311) 168 | - Install rsync (#301) 169 | 170 | ## [15.2.7] - 2020-07-17 171 | ### Added 172 | - Only allow authentication on a single registry (#298) 173 | - Allow user to specify sudo ssh user (#290) 174 | - Rely on "bootstrap" to configure MGR module (#270) 175 | ### Removed 176 | - Drop unqualified image name support (#299) 177 | ### Fixed 178 | - Check 'ceph orch status' output (#283) 179 | - Optimize 'ceph-salt status' (#293) 180 | - Improve "no minions matched" message when adding/removing minions (#287) 181 | 182 | ## [15.2.6] - 2020-07-01 183 | ### Added 184 | - Support registry authentication (#277) 185 | - Allow users to disable custom registries configuration (#262) 186 | - Allow to re-apply config (#269) 187 | - Add "Conflicts: deepsea" to spec file (#259) 188 | - Switch to reactive UI refresh after execution is complete (#254) 189 | - Allow user to "paused" UI refresh during execution (#254) 190 | - Report log file location on exit (#255) 191 | - Allow users to provide their dashboard cert for bootstrap (#253) 192 | ### Removed 193 | - Don't log scrollbar info when rendering scrollbar (#263) 194 | ### Fixed 195 | - Fix log location message (#278) 196 | - Reduce the number of remote grain requests (#271) 197 | - Optimize "ceph_orch.wait_for_admin_host" state (#276) 198 | - Handle execution errors on command line (#264) 199 | - Improve containers step/stage descriptions (#268) 200 | - Fix python3-ntplib required version (#249) 201 | 202 | ## [15.2.5] - 2020-05-25 203 | ### Added 204 | - Support SSH keys import and export (#243) 205 | - Probe external time servers (#247) 206 | - Persist journal logs (#244) 207 | - Add 'cephadm' role (#235) 208 | ### Fixed 209 | - Retry first chronyc execution (#239) 210 | 211 | ## [15.2.4] - 2020-05-15 212 | ### Added 213 | - Enforce dashboard password change upon first login (#220) 214 | - User feedback when adding/removing minions (#227) 215 | - Optimize "ceph-salt config" command (#224) 216 | - Optimize remote grain get (#230) 217 | - Persist default values in pillar data (#219) 218 | - Log warn when public IP is loopback IP (#216) 219 | ### Fixed 220 | - Fix error on 'ceph-salt status' when pillar data is empty (#221) 221 | - Don't log dashboard password (#228) 222 | - Unable to see dashboard password (#220) 223 | - Do not log pillar data secrets (#234) 224 | - Wait longer for clock sync (#233) 225 | 226 | ## [15.2.3] - 2020-05-04 227 | ### Added 228 | - Support time server not managed by ceph-salt (#206) 229 | - Sync clocks to avoid clock skew when MONs start (#202) 230 | ### Fixed 231 | - Wait for admin should fail if any admin failed (#207) 232 | - Store minion_id in pillar instead of hostname (#211) 233 | 234 | ## [15.2.2] - 2020-04-28 235 | ### Added 236 | - Advanced settings for "cephadm bootstrap" (#170) 237 | - Rename `ceph-salt deploy` to `ceph-salt apply` (#200) 238 | - Require Salt >= 3000 (#185) 239 | ### Removed 240 | - Remove "disable cephadm bootstrap" functionality (#184) 241 | ### Fixed 242 | - Fix `status` error when no minions are specified (#188) 243 | 244 | ## [15.2.1] - 2020-04-16 245 | ### Added 246 | - Support adding new hosts after initial deployment (#175) 247 | - Skip monitoring stack on bootstrap (#179) 248 | - Allow explicit set chrony subnet (#165) 249 | - Avoid 127.0.0.1 as a nodes public_ip (#174) 250 | - Allow users to configure custom registries (#113) 251 | - Rename minions "rm" command to "remove" (#167) 252 | - Use lowercase on config nodes (#166) 253 | - Support quoted string values (#162) 254 | - Support salt 3000 (#159) 255 | - Allow explicit set bootstrap Mon IP (#156) 256 | ### Removed 257 | - OSDs are no longer deployed by `ceph-salt` (#146) 258 | - Additional MONs and MGRs are no longer deployed by `ceph-salt` (#151) 259 | ### Fixed 260 | - Install private/public keys on admin nodes (#155) 261 | - Fix "status" error when ceph_orch salt module is not available (#157) 262 | 263 | ## [15.2.0] - 2020-03-25 264 | ### Added 265 | - Support bootstrap ceph config (#129) 266 | - Run "cephadm check-host" on all minions (#137) 267 | ### Fixed 268 | - Improve descriptions of stages and steps (#144) 269 | - Do not omit bootstrap MGR from "ceph orch apply mgr" (#136) 270 | - Use new OSD creation syntax (#133) 271 | - Work around podman/runc bug (#134) 272 | 273 | ## [15.1.1] - 2020-03-18 274 | ### Added 275 | - Add "Admin" role (#121) 276 | - Support config export and import (#90) 277 | - Add "status" command (#112) 278 | - Automatically set chooseleaf type if needed (#105) 279 | - Improve error handling when calling salt commands (#89) 280 | - Rename ceph-bootstrap to ceph-salt (#93) 281 | ### Fixed 282 | - Use `cephadm pull` instead of `podman pull` (#122) 283 | - Handle execution errors (#126) 284 | - Fix error when deploying additional mgrs (#119) 285 | - Bump PyYAML dependency (#117) 286 | - No default value for Ceph container image path (#115) 287 | - Work around timing issue in cephadm device list (#109) 288 | - Renamed "host" field to "hostname" (#111) 289 | - Use "ceph orch daemon add mon" to add remaining MONs (#108) 290 | - Tell ceph orch the right number of mgrs (#106) 291 | - Add `--skip-prepare-host` to `cephadm bootstrap` (#98) 292 | - Fix salt job return event processing (#95) 293 | - Check os_family before executing zypper command (#92) 294 | - Eliminate implicit dependency on which (#85) 295 | ### Removed 296 | - Migrate ceph-bootstrap-qa to sesdev (#88) 297 | 298 | ## [15.1.0] - 2020-02-17 299 | ### Added 300 | - System update and reboot during deployment (#11) 301 | - Ensure ceph-salt-formula is loaded by the salt-master before deploy (#65) 302 | - Automatic pillar setup (#8) 303 | - Check salt-master is up and running (#61) 304 | - Wait more verbosely on QA ceph_health_test (#62) 305 | ### Fixed 306 | - Rename calls to Ceph Orchestrator Apply (#80) 307 | - Rename calls to Ceph Orchestrator (#73) 308 | - Explicitly install podman (#72) 309 | 310 | ## [15.0.2] - 2020-01-29 311 | ### Added 312 | - New "deploy" command with real-time feedback (#9) 313 | - Use salt-event bus to notify about execution progress (#30) 314 | - Initial integration testing (#33) 315 | ### Fixed 316 | - Require root privileges (#18) 317 | - Remove salt python API terminal output (#10) 318 | - Hide Dashboard password (#48) 319 | - Fixed error when deploying without any role (#45) 320 | - Fixed error when deploying without any time server (#40) 321 | - Fixed bootstrap help message (#36) 322 | 323 | ## [15.0.1] - 2020-01-17 324 | ### Added 325 | - Each config shell command now returns a success or error message (#13) 326 | - Moved ceph-salt-formula into ceph-bootstrap project as a subpackage (#26) 327 | ### Fixed 328 | - Check if minion FQDN resolves to loopback IP address (#21) 329 | - Fixed "help" command when help text is not provided (#14) 330 | - Fixed "bootstrap_mon" update when the last MON is removed (#17) 331 | - Minions without role are also added to "ceph-salt:minions:all" (#22) 332 | - Fix minion removal upon error (#24) 333 | 334 | ## [0.1.0] - 2019-12-12 335 | ### Added 336 | - Mgr/Mon roles configuration 337 | - Configuration of drive groups specifications to be used in OSD deployment 338 | - Ceph-dashboard credentials configuration 339 | - Ceph daemon container image path configuration 340 | - Control Mon/Mgr/OSD deployment with enable/disable flags 341 | 342 | ## [0.0.1] - 2019-12-03 343 | ### Added 344 | - `sesboot`: CLI tool 345 | - RPM spec file. 346 | - Minimal README. 347 | - The CHANGELOG file. 348 | 349 | [Unreleased]: https://github.com/ceph/ceph-salt/compare/v16.2.5...HEAD 350 | [16.2.5]: https://github.com/ceph/ceph-salt/releases/tag/v16.2.5 351 | [16.2.4]: https://github.com/ceph/ceph-salt/releases/tag/v16.2.4 352 | [16.2.3]: https://github.com/ceph/ceph-salt/releases/tag/v16.2.3 353 | [16.2.2]: https://github.com/ceph/ceph-salt/releases/tag/v16.2.2 354 | [16.2.1]: https://github.com/ceph/ceph-salt/releases/tag/v16.2.1 355 | [16.2.0]: https://github.com/ceph/ceph-salt/releases/tag/v16.2.0 356 | [15.2.16]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.16 357 | [15.2.15]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.15 358 | [15.2.14]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.14 359 | [15.2.13]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.13 360 | [15.2.12]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.12 361 | [15.2.11]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.11 362 | [15.2.10]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.10 363 | [15.2.9]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.9 364 | [15.2.8]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.8 365 | [15.2.7]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.7 366 | [15.2.6]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.6 367 | [15.2.5]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.5 368 | [15.2.4]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.4 369 | [15.2.3]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.3 370 | [15.2.2]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.2 371 | [15.2.1]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.1 372 | [15.2.0]: https://github.com/ceph/ceph-salt/releases/tag/v15.2.0 373 | [15.1.1]: https://github.com/ceph/ceph-salt/releases/tag/v15.1.1 374 | [15.1.0]: https://github.com/ceph/ceph-salt/releases/tag/v15.1.0 375 | [15.0.2]: https://github.com/ceph/ceph-salt/releases/tag/v15.0.2 376 | [15.0.1]: https://github.com/ceph/ceph-salt/releases/tag/v15.0.1 377 | [0.1.0]: https://github.com/ceph/ceph-salt/releases/tag/v0.1.0 378 | [0.0.1]: https://github.com/ceph/ceph-salt/releases/tag/v0.0.1 379 | --------------------------------------------------------------------------------