├── .gitignore ├── test └── state_top.sls ├── ceph ├── osmap.yaml ├── repo │ ├── macros.jinja │ ├── init.sls │ └── map.jinja ├── defaults.yaml ├── mon.sls ├── init.sls ├── mgr.sls ├── osfamilymap.yaml ├── map.jinja ├── codenamemap.yaml └── osd.sls ├── LICENSE ├── pillar.example ├── Vagrantfile └── README.rst /.gitignore: -------------------------------------------------------------------------------- 1 | top.sls 2 | *log 3 | .vagrant 4 | -------------------------------------------------------------------------------- /test/state_top.sls: -------------------------------------------------------------------------------- 1 | base: 2 | '*': 3 | - ceph 4 | - ceph.mon 5 | -------------------------------------------------------------------------------- /ceph/osmap.yaml: -------------------------------------------------------------------------------- 1 | {%- from tpldir + '/repo/map.jinja' import repo with context %} 2 | 3 | Fedora: 4 | pkg_repo: 5 | baseurl: '{{ repo.official }}/rpm-{{ repo.release }}/{{ repo.oscode }}/$basearch' 6 | 7 | # vim: ft=sls 8 | -------------------------------------------------------------------------------- /ceph/repo/macros.jinja: -------------------------------------------------------------------------------- 1 | {%- from "ceph/map.jinja" import ceph with context -%} 2 | 3 | {%- macro format_kwargs(kwarg) -%} 4 | {%- filter indent(4) %} 5 | {%- for k, v in kwarg|dictsort() %} 6 | - {{ k }}: {{ v }} 7 | {%- endfor %} 8 | {%- endfilter %} 9 | {%- endmacro %} 10 | 11 | # vim: ft=sls 12 | -------------------------------------------------------------------------------- /ceph/defaults.yaml: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=yaml 3 | 4 | ceph: 5 | release: nautilus 6 | use_upstream_repo: true 7 | fromrepo: '' 8 | oscode: '' 9 | packages: 10 | - ceph 11 | cluster_name: ceph 12 | config: 13 | file: /etc/ceph/ceph.conf 14 | global: 15 | authentication_type: 'cephx' 16 | mon_hosts: [] 17 | osds: 18 | active: [] 19 | removed: [] 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013-2015 Salt Stack Formulas 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /pillar.example: -------------------------------------------------------------------------------- 1 | ceph: 2 | release: nautilus 3 | use_upstream_repo: false 4 | #oscode: OpenSUSE_Fumbleweed 5 | 6 | config: 7 | file: /etc/ceph/ceph.conf 8 | global: 9 | # Set cluster id, use command 'uuidgen' to generate it 10 | fsid: 0fffffff-35be-40a0-a76e-fff899cb85da 11 | authentication_type: 'none' 12 | 13 | # Ceph mon list 14 | mon_hosts: 15 | - 10.0.1.10 16 | - 10.0.1.11 17 | 18 | osds: 19 | # Active OSD device list 20 | active: 21 | - /dev/sdc 22 | - /dev/sdd 23 | # if you want to specify separate journal device 24 | - /dev/sdj:/dev/sdb1 # data-path:journal-path 25 | removed: 26 | - /dev/sde 27 | 28 | packages: 29 | - apt-transport-https 30 | -------------------------------------------------------------------------------- /ceph/mon.sls: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=yaml 3 | 4 | {% from "ceph/map.jinja" import ceph with context -%} 5 | 6 | {% set mon_name = grains['host'] %} 7 | {% set mon_dir = '/var/lib/ceph/mon/' ~ ceph.cluster_name ~ '-' ~ mon_name %} 8 | 9 | mkdir_dir_for_{{ mon_name }}: 10 | file.directory: 11 | - name: {{ mon_dir }} 12 | - user: ceph 13 | - group: ceph 14 | 15 | add_mon_{{ mon_name }}: 16 | cmd.run: 17 | - name: "ceph-mon --cluster {{ ceph.cluster_name }} --setuser ceph --setgroup ceph --mkfs --id {{ mon_name }} --keyring /dev/null" 18 | - unless: 'test -d {{ mon_dir }}/store.db' 19 | 20 | start_mon_service_for_{{ mon_name }}: 21 | service.running: 22 | - name: ceph-mon@{{ mon_name }} 23 | - enable: True 24 | -------------------------------------------------------------------------------- /ceph/init.sls: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=yaml 3 | 4 | {% from "ceph/map.jinja" import ceph with context -%} 5 | 6 | include: 7 | - ceph.repo 8 | 9 | install_ceph_pkgs: 10 | pkg.installed: 11 | - pkgs: {{ ceph.packages | json }} 12 | 13 | create_ceph_config_file: 14 | file.touch: 15 | - name: {{ ceph.config.file }} 16 | - unless: test -e {{ ceph.config.file }} 17 | 18 | ceph_config_file: 19 | ini.options_present: 20 | - name: {{ ceph.config.file }} 21 | - sections: 22 | global: 23 | {{ ceph.config.global | json }} 24 | 25 | ceph_config_mon_host: 26 | ini.options_present: 27 | - name: {{ ceph.config.file }} 28 | - sections: 29 | global: 30 | mon_host: {{ ceph.mon_hosts|join(', ') }} 31 | -------------------------------------------------------------------------------- /ceph/mgr.sls: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=yaml 3 | 4 | {% from "ceph/map.jinja" import ceph with context -%} 5 | 6 | {% set mgr_name = grains['host'] %} 7 | {% set mgr_dir = '/var/lib/ceph/mgr' %} 8 | 9 | mkdir_mgr_for_{{ mgr_name }}: 10 | file.directory: 11 | - name: {{ mgr_dir }} 12 | - user: ceph 13 | - group: ceph 14 | 15 | mgr_auth_key_{{ mgr_name }}: 16 | cmd.run: 17 | - name: "ceph auth get-or-create mgr.{{ ceph.cluster_name }} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > {{ mgr_dir }}/ceph-{{ ceph.cluster_name }}" 18 | - unless: 'test -e {{ mgr_dir }}/ceph-{{ ceph.cluster_name }}' 19 | 20 | start_mgr_service_for_{{ mgr_name }}: 21 | service.running: 22 | - name: ceph-mgr@{{ mgr_name }} 23 | - enable: True 24 | -------------------------------------------------------------------------------- /ceph/repo/init.sls: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=jinja 3 | 4 | {%- from 'ceph/map.jinja' import ceph with context -%} 5 | {%- from tpldir + "/macros.jinja" import format_kwargs with context -%} 6 | 7 | {% if 'pkg_repo' in ceph %} 8 | {% if ceph.use_upstream_repo == true %} 9 | 10 | # Add upstream repository for your distro 11 | ceph-repo: 12 | pkgrepo.managed: 13 | {{- format_kwargs(ceph.pkg_repo) }} 14 | 15 | {%- else -%} 16 | 17 | # Remove repo configuration (and GnuPG key) 18 | ceph-repo: 19 | pkgrepo.absent: 20 | - name: {{ ceph.pkg_repo.name }} 21 | {%- if 'pkg_repo_keyid' in ceph %} 22 | - keyid: {{ ceph.pkg_repo_keyid }} 23 | {%- endif %} 24 | 25 | {%- endif -%} 26 | {%- elif grains.os not in ('Windows', 'MacOS',) %} 27 | 28 | ceph-repo: 29 | test.show_notification: 30 | - text: | 31 | Ceph does not provide package repository for {{ grains.os_family }} 32 | 33 | {%- endif %} 34 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | config.vm.box = "ubuntu/xenial64" 6 | config.vm.synced_folder "./", "/srv/salt", id: "vagrant-root" 7 | 8 | config.vm.provider "virtualbox" do |vb| 9 | vb.name = "ceph_ci" 10 | vb.memory = "1024" 11 | end 12 | 13 | config.vm.provision "shell", inline: <<-SHELL 14 | if [ ! -h /srv/salt/top.sls ] ; then ln -s /srv/salt/test/state_top.sls /srv/salt/top.sls ; fi 15 | SHELL 16 | config.vm.provision :salt do |salt| 17 | salt.masterless = true 18 | salt.pillar({ 19 | "ceph" => { 20 | "config" => { 21 | "global" => { 22 | "fsid" => "0fffffff-35be-40a0-a76e-fff899cb85da", 23 | "authentication_type" => "none", 24 | }, 25 | }, 26 | "mon_hosts" => [ 27 | "127.0.0.1", 28 | ], 29 | } 30 | }) 31 | salt.colorize = true 32 | salt.verbose = true 33 | salt.log_level = "warning" 34 | salt.run_highstate = true 35 | end 36 | 37 | end 38 | -------------------------------------------------------------------------------- /ceph/osfamilymap.yaml: -------------------------------------------------------------------------------- 1 | {%- from tpldir + '/repo/map.jinja' import repo with context %} 2 | 3 | {%- if grains.os_family not in ('FreeBSD',) and 'oscodename' in grains %} 4 | 5 | Debian: 6 | pkg_repo: 7 | name: deb {{ repo.official }}/debian-{{ repo.release }}/ {{ grains.oscodename }} main 8 | humanname: Ceph {{ repo.release }} Debian Repository 9 | key_url: '{{ repo.official }}/keys/release.asc' 10 | file: /etc/apt/sources.list.d/ceph.list 11 | 12 | RedHat: 13 | pkg_repo: 14 | name: ceph-redhat-{{ repo.release }} 15 | humanname: Ceph {{ repo.release }} $releasever - $basearch 16 | gpgcheck: 1 17 | gpgkey: '{{ repo.official }}/keys/release.asc' 18 | baseurl: {{ repo.official }}/rpm-{{ repo.release }}/el{{ '' if 'osrelease_info' not in grains else grains.osrelease_info[0] }}/$basearch 19 | 20 | Suse: 21 | oscode: openSUSE_Tumbleweed 22 | pkg_repo: 23 | name: ceph-opensuse-{{ repo.release }} 24 | humanname: Ceph {{ repo.release }} $releasever - $basearch 25 | baseurl: '{{ repo.suse }}:/ceph:/{{ repo.release }}/{{ repo.oscode }}' 26 | key_url: '{{ repo.suse }}:/ceph:/{{ repo.release }}/{{ repo.oscode }}/repodata/repomd.xml.key' 27 | gpgcheck: 1 28 | gpgautoimport: True 29 | 30 | {%- endif %} 31 | 32 | # vim: ft=sls 33 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | ceph-formula 3 | ============ 4 | 5 | Formulas to set up and configure a Ceph cluster. 6 | 7 | .. note:: 8 | 9 | See the full `Salt Formulas installation and usage instructions 10 | `_. 11 | 12 | Available states 13 | ================ 14 | 15 | .. contents:: 16 | :local: 17 | 18 | ``ceph`` 19 | ---------- 20 | 21 | Install and configure Ceph 22 | 23 | ``ceph.mon`` 24 | ---------- 25 | 26 | Install and configure Ceph monitor. 27 | 28 | ``ceph.osd`` 29 | ---------- 30 | 31 | Install and configure Ceph OSDs. 32 | 33 | 34 | ``ceph.repo`` 35 | --------------------- 36 | 37 | Configures the official Ceph (upstream) repository on target system (either 38 | `download.ceph.org` or `www.suse.com` mirror). 39 | 40 | The state relies on ``ceph:use_upstream_repo`` pillar boolean value- 41 | 42 | * ``True`` (default): adds the upstream repository to install packages from. 43 | * ``False``: makes sure that the repository configuration is absent. 44 | 45 | The ``ceph:release`` pillar controls which release to install. Defaults to ``luminous``. 46 | 47 | 48 | 49 | Usage 50 | ======== 51 | 52 | For a list of all available options, look at: *ceph/defaults.yaml* - also have a look at the *pillar.example* and *map.jinja*. 53 | 54 | Supports GNU Linux (Ubuntu, Fedora, Centos, Suse) 55 | -------------------------------------------------------------------------------- /ceph/map.jinja: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=jinja 3 | 4 | {% import_yaml "ceph/defaults.yaml" as defaults %} 5 | {% import_yaml "ceph/osfamilymap.yaml" as osfamilymap %} 6 | {% import_yaml "ceph/osmap.yaml" as osmap %} 7 | {% import_yaml "ceph/codenamemap.yaml" as oscodenamemap %} 8 | 9 | {% set ceph = salt['grains.filter_by']( 10 | defaults, 11 | merge=salt['grains.filter_by']( 12 | osfamilymap, 13 | grain='os_family', 14 | merge=salt['grains.filter_by']( 15 | osmap, 16 | grain='os', 17 | merge=salt['grains.filter_by']( 18 | oscodenamemap, 19 | grain='oscodename', 20 | merge=salt['pillar.get']('ceph', {}), 21 | ), 22 | ), 23 | ), 24 | base='ceph', 25 | ) %} 26 | 27 | {% if ceph.config.global.authentication_type == 'cephx' %} 28 | {% set auth_type = 'cephx' %} 29 | {% else %} 30 | {% set auth_type = 'none' %} 31 | {% endif %} 32 | 33 | {% do ceph.config.global.update( 34 | { 'auth_cluster_required': auth_type, 35 | 'auth_service_required': auth_type, 36 | 'auth_client_required': auth_type, 37 | 'auth_supported': auth_type 38 | }) 39 | %} 40 | 41 | {% if ceph.cluster_name %} 42 | {% do ceph.config.update({'file': '/etc/ceph/' + ceph.cluster_name + '.conf', }) %} 43 | {% endif %} 44 | 45 | # make 'ceph' alias for 'ceph' 46 | {% set ceph = ceph %} 47 | -------------------------------------------------------------------------------- /ceph/codenamemap.yaml: -------------------------------------------------------------------------------- 1 | {%- from tpldir + '/repo/map.jinja' import repo with context %} 2 | 3 | {% macro debian_codename(name, release, codename=none) %} 4 | {% if repo.use_upstream_repo == true %} 5 | {% set release = repo.release %} 6 | {% endif %} 7 | {{ codename|default(name, true) }}: 8 | fromrepo: {{ repo.fromrepo|default(name, true) }} 9 | pkg_repo: 10 | name: 'deb {{ repo.official }}/debian-{{ release }}/ {{ name }} main' 11 | {% endmacro %} 12 | 13 | 14 | {% macro redhat_codename(name, release, codename=none) %} 15 | {% if repo.use_upstream_repo == true %} 16 | {% set release = repo.release %} 17 | {% endif %} 18 | {{ codename|default(name, true) }}: 19 | fromrepo: {{ repo.fromrepo|default(name, true) }} 20 | pkg_repo: 21 | baseurl: '{{ repo.official }}/rpm-{{ release }}/$releasever' 22 | {% endmacro %} 23 | 24 | ## oscodename mappings 25 | {{ debian_codename('wheezy', 'firefly') }} 26 | {{ debian_codename('wheezy', 'firefly', 'Debian GNU/Linux 7 (wheezy)') }} 27 | {{ debian_codename('jessie', 'jewel') }} #backports 28 | {{ debian_codename('jessie', 'jewel', 'Debian GNU/Linux 8 (jessie)') }} #backports 29 | {{ debian_codename('stretch', 'jewel') }} 30 | {{ debian_codename('stretch', 'jewel', 'Debian GNU/Linux 9 (stretch)') }} 31 | 32 | {{ debian_codename('trusty', 'firefly') }} 33 | {{ debian_codename('xenial', 'jewel') }} 34 | {{ debian_codename('artful', 'luminous') }} 35 | {{ debian_codename('bionic', 'luminous') }} 36 | {{ debian_codename('cosmic', 'luminous') }} 37 | 38 | {{ redhat_codename('Fedora-27', 'luminous', 'Fedora 27 (Twenty Seven)') }} 39 | {{ redhat_codename('Fedora-26', 'luminous', 'Fedora 26 (Twenty Six)') }} 40 | 41 | # vim: ft=sls 42 | -------------------------------------------------------------------------------- /ceph/repo/map.jinja: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=jinja 3 | 4 | {% import_yaml "ceph/defaults.yaml" as defaults %} 5 | 6 | {% set use_upstream_repo = salt['pillar.get']('ceph:use_upstream_repo', defaults.ceph.use_upstream_repo) %} 7 | {% set release = salt['pillar.get']('ceph:release', defaults.ceph.release) %} 8 | {% set fromrepo = salt['pillar.get']('ceph:fromrepo', defaults.ceph.fromrepo) %} 9 | {% set user = salt['pillar.get']('ceph.user', None) %} 10 | {% set group = salt['pillar.get']('ceph.group', None) %} 11 | 12 | {%- if 'oscodename' in grains %} 13 | {%- if grains.os_family == 'Suse' %} 14 | {% set oscode = salt['pillar.get']('ceph.oscode','openSUSE_Tumbleweed') | replace(' ','_') %} 15 | {%- elif grains.os_family not in ('FreeBSD',) %} 16 | {% set oscode = salt['pillar.get']('ceph.oscode', salt['grains.get']('oscodename')) | replace(' ','_') %} 17 | {%- endif %} 18 | {%- endif %} 19 | 20 | {% if grains.os == 'MacOS' %} 21 | {% set user = salt['pillar.get']('ceph.user') or salt['cmd.run']("stat -f '%Su' /dev/console") %} 22 | {% set group = salt['pillar.get']('ceph.group') or salt['cmd.run']("stat -f '%Sg' /dev/console") %} 23 | {% endif %} 24 | 25 | {% set official = 'https://download.ceph.com' %} 26 | {% set suse = 'https://download.opensuse.org/repositories/filesystems' %} 27 | 28 | {%- set repo = {} %} 29 | {%- do repo.update( { 'use_upstream_repo' : use_upstream_repo, 30 | 'release' : release, 31 | 'fromrepo' : fromrepo, 32 | 'oscode' : oscode, 33 | 'user' : user, 34 | 'group' : group, 35 | 'official' : official, 36 | 'suse' : suse, 37 | } ) %} 38 | 39 | # vim: ft=jinja 40 | -------------------------------------------------------------------------------- /ceph/osd.sls: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=yaml 3 | 4 | {% from "ceph/map.jinja" import ceph with context -%} 5 | 6 | {% for active_osd in ceph.osds.active %} 7 | {% set osd_params = active_osd.split(':') %} 8 | {% set data_path = osd_params[0] %} 9 | {% if osd_params|length == 2 %} 10 | {% set journal_path = osd_params[1] %} 11 | {% else %} 12 | {% set journal_path = '' %} 13 | {% endif %} 14 | 15 | zap_disk_{{ data_path }}: 16 | cmd.run: 17 | - name: 'ceph-disk zap {{ data_path }}' 18 | - unless: "ceph-disk list | grep -E ' *{{ data_path }}1? .*ceph data, (prepared|active)'" 19 | 20 | prepare_osd_device_{{ data_path }}: 21 | cmd.run: 22 | - name: 'ceph-disk prepare --cluster {{ ceph.cluster_name }} {{ data_path }} {{ journal_path }}' 23 | - unless: "ceph-disk list | grep -E ' *{{ data_path }}1? .*ceph data, (prepared|active)'" 24 | 25 | {% endfor %} 26 | 27 | {% for removed_osd in ceph.osds.removed %} 28 | {% set osd_id = salt['cmd.run']("ceph-disk list | grep '" ~ removed_osd ~ "1 ceph' | awk -F'[.,]' '{print $5}'") %} 29 | 30 | osd_out_{{ removed_osd }}: 31 | cmd.run: 32 | - name: 'ceph osd out {{ osd_id }}' 33 | - unless: "ceph-disk list | grep -E ' *{{ removed_osd }}1? .*ceph data, unprepared'" 34 | 35 | stop_service_{{ removed_osd }}: 36 | service.dead: 37 | - name: ceph-osd@{{ osd_id }} 38 | - unless: "ceph-disk list | grep -E ' *{{ removed_osd }}1? .*ceph data, unprepared'" 39 | 40 | crush_remove_{{ removed_osd }}: 41 | cmd.run: 42 | - name: 'ceph --cluster {{ ceph.cluster_name }} osd crush remove osd.{{ osd_id }}' 43 | - unless: "ceph-disk list | grep -E ' *{{ removed_osd }}1? .*ceph data, unprepared'" 44 | 45 | auth_del_{{ removed_osd }}: 46 | cmd.run: 47 | - name: 'ceph --cluster {{ ceph.cluster_name }} auth del osd.{{ osd_id }}' 48 | - unless: "ceph-disk list | grep -E ' *{{ removed_osd }}1? .*ceph data, unprepared'" 49 | 50 | osd_rm_{{ removed_osd }}: 51 | cmd.run: 52 | - name: 'ceph --cluster {{ ceph.cluster_name }} osd rm {{ osd_id }}' 53 | - unless: "ceph-disk list | grep -E ' *{{ removed_osd }}1? .*ceph data, unprepared'" 54 | 55 | rm_umount_rm_{{ removed_osd }}: 56 | cmd.run: 57 | - name: 'rm -fr /var/lib/ceph/osd/{{ ceph.cluster_name }}-{{ osd_id }}/* && 58 | umount /var/lib/ceph/osd/{{ ceph.cluster_name }}-{{ osd_id }} && 59 | rm -fr /var/lib/ceph/osd/{{ ceph.cluster_name }}-{{ osd_id }}' 60 | - unless: "ceph-disk list | grep -E ' *{{ removed_osd }}1? .*ceph data, unprepared'" 61 | 62 | {% endfor %} 63 | --------------------------------------------------------------------------------