├── docs ├── source │ ├── _static │ │ └── .empty │ ├── contents.rst │ ├── releasing.rst │ ├── man │ │ └── index.rst │ ├── changelog.rst │ ├── _themes │ │ └── solarized.py │ ├── cluster.rst │ └── conf.py └── Makefile ├── ceph_installer ├── cli │ ├── __init__.py │ ├── constants.py │ ├── log.py │ ├── main.py │ ├── dev.py │ ├── util.py │ ├── decorators.py │ └── task.py ├── tests │ ├── __init__.py │ ├── controllers │ │ ├── test_calamari.py │ │ ├── test_status.py │ │ ├── test_setup.py │ │ ├── test_rgw.py │ │ ├── test_errors.py │ │ ├── test_agent.py │ │ └── test_tasks.py │ ├── config.py │ ├── test_process.py │ ├── test_hooks.py │ ├── test_tasks.py │ ├── conftest.py │ └── test_util.py ├── commands │ ├── __init__.py │ └── populate.py ├── __init__.py ├── controllers │ ├── calamari.py │ ├── status.py │ ├── tasks.py │ ├── root.py │ ├── __init__.py │ ├── agent.py │ ├── setup.py │ ├── errors.py │ ├── osd.py │ ├── rgw.py │ └── mon.py ├── async.py ├── app.py ├── models │ ├── tasks.py │ └── __init__.py ├── templates.py ├── process.py ├── tasks.py ├── hooks.py └── schemas.py ├── tests └── functional │ ├── nightly-centos7 │ ├── Vagrantfile │ ├── test.yml │ ├── hosts │ ├── group_vars │ │ └── all │ └── vagrant_variables.yml │ ├── nightly-xenial │ ├── Vagrantfile │ ├── hosts │ ├── group_vars │ │ └── all │ ├── vagrant_variables.yml │ └── test.yml │ ├── .gitignore │ ├── requirements.txt │ ├── playbooks │ ├── roles │ │ └── installer │ │ │ ├── defaults │ │ │ └── main.yml │ │ │ ├── templates │ │ │ └── dev_repos.j2 │ │ │ └── tasks │ │ │ └── main.yml │ └── setup.yml │ ├── scripts │ └── generate_ssh_config.sh │ └── tox.ini ├── bin ├── ceph-installer-celery ├── ceph-installer-gunicorn └── ceph-installer ├── deploy └── playbooks │ ├── roles │ ├── common │ │ ├── defaults │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── ceph-installer.sysconfig.j2 │ │ │ ├── ceph-installer-celery.service.j2 │ │ │ └── ceph-installer.service.j2 │ │ ├── handlers │ │ │ └── main.yml │ │ ├── vars │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── systemd.yml │ │ │ └── main.yml │ └── celery │ │ └── tasks │ │ └── main.yml │ ├── ansible.cfg │ └── deploy.yml ├── MANIFEST.in ├── systemd ├── ceph-installer.sysconfig ├── 80-ceph-installer.preset ├── ceph-installer-celery.service └── ceph-installer.service ├── .gitchangelog.rc ├── firewalld └── ceph-installer.xml ├── README.rst ├── tox.ini ├── .gitignore ├── selinux ├── ceph_installer.fc ├── ceph_installer.if └── ceph_installer.te ├── rpm ├── README.rst └── ktdreyer-ceph-installer.cfg ├── LICENSE ├── Makefile ├── config └── config.py ├── setup.py └── ceph-installer.spec.in /docs/source/_static/.empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ceph_installer/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ceph_installer/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ceph_installer/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/functional/nightly-centos7/Vagrantfile: -------------------------------------------------------------------------------- 1 | ../Vagrantfile -------------------------------------------------------------------------------- /tests/functional/nightly-xenial/Vagrantfile: -------------------------------------------------------------------------------- 1 | ../Vagrantfile -------------------------------------------------------------------------------- /ceph_installer/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | __version__ = '1.3.1' 4 | -------------------------------------------------------------------------------- /tests/functional/nightly-centos7/test.yml: -------------------------------------------------------------------------------- 1 | ../nightly-xenial/test.yml -------------------------------------------------------------------------------- /tests/functional/.gitignore: -------------------------------------------------------------------------------- 1 | ubuntu-key/ 2 | fetch/ 3 | vagrant_ssh_config 4 | disk-* 5 | *.retry 6 | -------------------------------------------------------------------------------- /bin/ceph-installer-celery: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | /usr/bin/celery -A async worker --loglevel=info 4 | -------------------------------------------------------------------------------- /deploy/playbooks/roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | app_home: /opt/{{ app_name }} 3 | app_use_ssl: yes 4 | -------------------------------------------------------------------------------- /deploy/playbooks/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | retry_files_enabled = False 3 | 4 | [ssh_connection] 5 | pipelining=True 6 | -------------------------------------------------------------------------------- /bin/ceph-installer-gunicorn: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | /usr/bin/gunicorn_pecan -w 10 -t 300 /etc/ceph-installer/config.py 4 | -------------------------------------------------------------------------------- /ceph_installer/cli/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | server_address = os.environ.get('CEPH_INSTALLER_ADDRESS', 'http://localhost:8181/') 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | graft config 4 | graft docs 5 | graft firewalld 6 | graft selinux 7 | graft systemd 8 | -------------------------------------------------------------------------------- /systemd/ceph-installer.sysconfig: -------------------------------------------------------------------------------- 1 | PECAN_CONFIG = /etc/ceph-installer/config.py 2 | CEPH_PLAYBOOK = /usr/share/ceph-ansible/site.yml.sample 3 | -------------------------------------------------------------------------------- /bin/ceph-installer: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from ceph_installer.cli import main 4 | 5 | if __name__ == '__main__': 6 | main.CephInstaller() 7 | -------------------------------------------------------------------------------- /.gitchangelog.rc: -------------------------------------------------------------------------------- 1 | tag_filter_regexp = r'^v[0-9]+\.[0-9]+(\.[0-9]+)?$' 2 | ignore_regexps = [ r'^version [0-9]+\.[0-9]+(\.[0-9]+)?.' ] 3 | include_merge = False 4 | -------------------------------------------------------------------------------- /tests/functional/requirements.txt: -------------------------------------------------------------------------------- 1 | # 1.6.1 fails with 'testinfra is in an unsupported or invalid wheel' 2 | # see https://github.com/philpep/testinfra/issues/201 3 | testinfra==1.6.0 4 | pytest-xdist 5 | -------------------------------------------------------------------------------- /tests/functional/playbooks/roles/installer/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | installer_dev_branch: master 4 | installer_dev_commit: latest 5 | ceph_ansible_dev_branch: master 6 | ceph_ansible_dev_commit: latest 7 | -------------------------------------------------------------------------------- /deploy/playbooks/deploy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: all 4 | roles: 5 | - common 6 | - celery 7 | vars: 8 | ansible_ssh_user: 'vagrant' 9 | app_name: "ceph-installer" 10 | branch: "master" 11 | -------------------------------------------------------------------------------- /systemd/80-ceph-installer.preset: -------------------------------------------------------------------------------- 1 | # ceph-installer web service 2 | # 3 | # Ensure that ceph-installer starts automatically when user installs the 4 | # package. This saves the user a step. 5 | 6 | enable ceph-installer.service 7 | -------------------------------------------------------------------------------- /deploy/playbooks/roles/common/templates/ceph-installer.sysconfig.j2: -------------------------------------------------------------------------------- 1 | PECAN_CONFIG={{ app_home }}/src/ceph-installer/config/config.py 2 | CEPH_PLAYBOOK={{ app_home }}/ceph-ansible/site.yml.sample 3 | HOME=/home/{{ ansible_ssh_user }} 4 | -------------------------------------------------------------------------------- /ceph_installer/tests/controllers/test_calamari.py: -------------------------------------------------------------------------------- 1 | 2 | class TestCalamariController(object): 3 | 4 | def test_index_get(self, session): 5 | result = session.app.get("/api/calamari/") 6 | assert result.status_int == 200 7 | -------------------------------------------------------------------------------- /deploy/playbooks/roles/celery/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install rabbitmq 3 | sudo: yes 4 | package: 5 | name: rabbitmq-server 6 | state: present 7 | 8 | - name: ensure rabbitmq is running and enabled 9 | sudo: yes 10 | service: 11 | name: rabbitmq-server 12 | state: started 13 | enabled: yes 14 | -------------------------------------------------------------------------------- /firewalld/ceph-installer.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Ceph Installer HTTP service 4 | The service allows you to install and configure ceph via a RestFul API that accepts and returns JSON responses along with meaningful error codes and messages. 5 | 6 | 7 | -------------------------------------------------------------------------------- /deploy/playbooks/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: reload systemd 4 | sudo: yes 5 | command: systemctl daemon-reload 6 | 7 | - name: restart app 8 | sudo: yes 9 | service: name=ceph-installer state=restarted enabled=yes 10 | 11 | - name: restart ceph-installer-celery 12 | sudo: yes 13 | service: name=ceph-installer-celery state=restarted enabled=yes 14 | -------------------------------------------------------------------------------- /tests/functional/scripts/generate_ssh_config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Generate a custom ssh config from Vagrant so that it can then be used by 3 | # ansible.cfg 4 | 5 | path=$1 6 | 7 | if [ $# -eq 0 ] 8 | then 9 | echo "A path to the scenario is required as an argument and it wasn't provided" 10 | exit 1 11 | fi 12 | 13 | cd "$path" 14 | vagrant ssh-config > vagrant_ssh_config 15 | -------------------------------------------------------------------------------- /deploy/playbooks/roles/common/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | system_packages: 4 | - python-devel 5 | - git 6 | - python-virtualenv 7 | - gcc 8 | - python-pip 9 | - vim 10 | - rabbitmq-server 11 | - libsemanage-python 12 | - libffi-devel 13 | - openssl-devel 14 | - rabbitmq-server 15 | # Latest as of this writing is 2.2.0.0 in EPEL 16 | - ansible 17 | 18 | ssl_requirements: [] 19 | -------------------------------------------------------------------------------- /tests/functional/nightly-centos7/hosts: -------------------------------------------------------------------------------- 1 | [installer] 2 | client0 address=192.168.3.40 3 | 4 | [mons] 5 | mon0 address=192.168.3.10 6 | mon1 address=192.168.3.11 7 | 8 | [osds] 9 | osd0 address=192.168.3.100 10 | 11 | [rgws] 12 | rgw0 address=192.168.3.50 13 | 14 | # This group is needed because we 15 | # need to register all these nodes 16 | # with the ceph-installer 17 | [test_nodes:children] 18 | mons 19 | osds 20 | rgws 21 | -------------------------------------------------------------------------------- /tests/functional/nightly-xenial/hosts: -------------------------------------------------------------------------------- 1 | [installer] 2 | client0 address=192.168.3.40 3 | 4 | [mons] 5 | mon0 address=192.168.3.10 6 | mon1 address=192.168.3.11 7 | 8 | [osds] 9 | osd0 address=192.168.3.100 10 | 11 | [rgws] 12 | rgw0 address=192.168.3.50 13 | 14 | # This group is needed because we 15 | # need to register all these nodes 16 | # with the ceph-installer 17 | [test_nodes:children] 18 | mons 19 | osds 20 | rgws 21 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Ceph Installer 2 | ============== 3 | An HTTP API (previously known as the "mariner installer") to provision and 4 | control the deployment process of Ceph clusters. 5 | 6 | The service allows you to install and configure ceph via a RestFul API that 7 | accepts and returns JSON responses along with meaningful error codes and 8 | messages. 9 | 10 | Read the full documentation here: http://docs.ceph.com/ceph-installer/docs/ 11 | -------------------------------------------------------------------------------- /ceph_installer/controllers/calamari.py: -------------------------------------------------------------------------------- 1 | from pecan import expose 2 | 3 | 4 | class CalamariController(object): 5 | 6 | @expose('json') 7 | def index(self): 8 | # TODO: allow some autodiscovery here so that clients can see what is 9 | # available 10 | return dict() 11 | 12 | @expose('json') 13 | def install(self): 14 | return {} 15 | 16 | @expose('json') 17 | def configure(self): 18 | return {} 19 | -------------------------------------------------------------------------------- /tests/functional/playbooks/roles/installer/templates/dev_repos.j2: -------------------------------------------------------------------------------- 1 | [ktdreyer-ceph-installer] 2 | name=Copr repo for ceph-installer owned by ktdreyer 3 | baseurl=https://copr-be.cloud.fedoraproject.org/results/ktdreyer/ceph-installer/epel-7-$basearch/ 4 | type=rpm-md 5 | skip_if_unavailable=True 6 | gpgcheck=1 7 | gpgkey=https://copr-be.cloud.fedoraproject.org/results/ktdreyer/ceph-installer/pubkey.gpg 8 | repo_gpgcheck=0 9 | enabled=1 10 | enabled_metadata=1 11 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, flake8 3 | 4 | [testenv] 5 | deps= 6 | pytest 7 | commands=py.test -v {posargs:ceph_installer/tests} 8 | 9 | [testenv:docs] 10 | basepython=python 11 | changedir=docs/source 12 | deps= 13 | sphinx 14 | sphinxcontrib-httpdomain 15 | commands= 16 | sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html 17 | 18 | [testenv:flake8] 19 | deps=flake8 20 | commands=flake8 --select=F,E9 {posargs:ceph_installer} 21 | -------------------------------------------------------------------------------- /tests/functional/nightly-centos7/group_vars/all: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | fsid: "deedcb4c-a67a-4997-93a6-92149ad2622a" 4 | monitor_secret: "AQA7P8dWAAAAABAAH/tbiZQn/40Z8pr959UmEA==" 5 | public_network: "192.168.3.0/24" 6 | monitor_interface: "eth1" 7 | monitors: 8 | - host: "mon0" 9 | interface: "eth1" 10 | - host: "mon1" 11 | interface: "eth1" 12 | devices_dedicated_journal: {"/dev/sdb": "/dev/sdc"} 13 | devices_collocated_journal: ["/dev/sdd"] 14 | journal_size: 100 15 | -------------------------------------------------------------------------------- /tests/functional/nightly-xenial/group_vars/all: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | fsid: "deedcb4c-a67a-4997-93a6-92149ad2622a" 4 | monitor_secret: "AQA7P8dWAAAAABAAH/tbiZQn/40Z8pr959UmEA==" 5 | public_network: "192.168.3.0/24" 6 | monitor_interface: "eth1" 7 | monitors: 8 | - host: "mon0" 9 | interface: "eth1" 10 | - host: "mon1" 11 | interface: "eth1" 12 | devices_dedicated_journal: {"/dev/sdb": "/dev/sdc"} 13 | devices_collocated_journal: ["/dev/sdd"] 14 | journal_size: 100 15 | -------------------------------------------------------------------------------- /systemd/ceph-installer-celery.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=ceph installer celery service 3 | After=network.target rabbitmq-server.service 4 | Requires=rabbitmq-server.service 5 | 6 | [Service] 7 | Type=simple 8 | ExecStart=/usr/bin/ceph-installer-celery 9 | EnvironmentFile=-/etc/sysconfig/ceph-installer 10 | User=ceph-installer 11 | WorkingDirectory=/usr/lib/python2.7/site-packages/ceph_installer 12 | StandardOutput=journal 13 | StandardError=journal 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /docs/source/contents.rst: -------------------------------------------------------------------------------- 1 | ceph-installer contents 2 | ======================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | index.rst 8 | cluster.rst 9 | 10 | 11 | Development 12 | =========== 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | 17 | releasing.rst 18 | 19 | 20 | Changelog 21 | ========= 22 | 23 | .. toctree:: 24 | :maxdepth: 2 25 | 26 | changelog.rst 27 | 28 | ceph-installer CLI 29 | ================== 30 | 31 | .. toctree:: 32 | :maxdepth: 2 33 | 34 | man/index.rst 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | log 2 | .DS_Store 3 | .vagrant 4 | 5 | *.py[cod] 6 | 7 | # Config files 8 | alembic.ini 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Packages 14 | *.egg 15 | *.egg-info 16 | dist 17 | build 18 | eggs 19 | sdist 20 | develop-eggs 21 | .installed.cfg 22 | __pycache__ 23 | 24 | # Installer logs 25 | pip-log.txt 26 | 27 | # Unit test / coverage reports 28 | .coverage 29 | .tox 30 | .cache 31 | 32 | # celery 33 | celerybeat-schedule 34 | 35 | # test db 36 | marinertest.db 37 | 38 | ceph-installer.spec 39 | -------------------------------------------------------------------------------- /systemd/ceph-installer.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=ceph installer gunicorn service 3 | After=network.target ceph-installer-celery.service 4 | Requires=ceph-installer-celery.service 5 | 6 | [Service] 7 | Type=simple 8 | ExecStart=/usr/bin/ceph-installer-gunicorn 9 | EnvironmentFile=-/etc/sysconfig/ceph-installer 10 | User=ceph-installer 11 | WorkingDirectory=/usr/lib/python2.7/site-packages/ceph_installer/ 12 | StandardOutput=journal 13 | StandardError=journal 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /ceph_installer/controllers/status.py: -------------------------------------------------------------------------------- 1 | from pecan import response, expose 2 | from ceph_installer.hooks import system_checks, SystemCheckError 3 | 4 | 5 | class StatusController(object): 6 | 7 | @expose('json') 8 | def index(self): 9 | for check in system_checks: 10 | try: 11 | check() 12 | except SystemCheckError as system_error: 13 | response.status = 500 14 | return {'message': system_error.message} 15 | return dict(message="ok") 16 | -------------------------------------------------------------------------------- /docs/source/releasing.rst: -------------------------------------------------------------------------------- 1 | .. releasing: 2 | 3 | ceph-installer release process 4 | ============================== 5 | 6 | When you are ready to cut a new version: 7 | 8 | #. Modify ``docs/source/changelog.rst`` with the changes since the latest 9 | release. Optionally, the ``gitchangelog`` program can help you write this. 10 | 11 | #. Bump the version number in ``ceph_installer/__init__.py`` and commit your 12 | changes. 13 | :: 14 | 15 | python setup.py bump 16 | 17 | #. Tag and release to PyPI. 18 | :: 19 | 20 | python setup.py release 21 | -------------------------------------------------------------------------------- /deploy/playbooks/roles/common/templates/ceph-installer-celery.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=ceph installer celery service 3 | After=network.target rabbitmq-server.service 4 | Requires=rabbitmq-server.service 5 | 6 | [Service] 7 | Type=simple 8 | ExecStart={{ app_home }}/bin/celery -A async worker --loglevel=debug 9 | EnvironmentFile=/etc/sysconfig/ceph-installer 10 | User={{ ansible_ssh_user }} 11 | WorkingDirectory={{ app_home }}/src/ceph-installer/ceph_installer 12 | StandardOutput=journal 13 | StandardError=journal 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /deploy/playbooks/roles/common/templates/ceph-installer.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=ceph installer gunicorn service 3 | After=network.target ceph-installer-celery.service 4 | Requires=ceph-installer-celery.service 5 | 6 | [Service] 7 | Type=simple 8 | ExecStart={{ app_home }}/bin/gunicorn_pecan -w 10 -t 300 {{ app_home }}/src/{{ app_name }}/config/config.py 9 | EnvironmentFile=/etc/sysconfig/ceph-installer 10 | User={{ ansible_ssh_user }} 11 | WorkingDirectory={{ app_home }}/src/ceph-installer/ceph_installer/ 12 | StandardOutput=journal 13 | StandardError=journal 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /tests/functional/playbooks/setup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | gather_facts: True 4 | tasks: 5 | - name: write all nodes to /etc/hosts 6 | sudo: yes 7 | blockinfile: 8 | dest: /etc/hosts 9 | block: | 10 | {{ hostvars[item]["address"] }} {{ item }} 11 | marker: "# {mark} ANSIBLE MANAGED BLOCK {{ item }}" 12 | with_inventory_hostnames: all 13 | 14 | - hosts: installer 15 | gather_facts: False 16 | roles: 17 | - installer 18 | 19 | - hosts: test_nodes 20 | gather_facts: False 21 | vars: 22 | installer_address: "http://192.168.3.40:8181" 23 | tasks: 24 | - name: register the node with ceph-installer 25 | shell: "curl {{ installer_address}}/setup/ | sudo bash" 26 | -------------------------------------------------------------------------------- /selinux/ceph_installer.fc: -------------------------------------------------------------------------------- 1 | /usr/bin/ceph-installer -- gen_context(system_u:object_r:ceph_installer_exec_t,s0) 2 | /usr/bin/ceph-installer-celery -- gen_context(system_u:object_r:ceph_installer_exec_t,s0) 3 | /usr/bin/ceph-installer-gunicorn -- gen_context(system_u:object_r:ceph_installer_exec_t,s0) 4 | 5 | /usr/lib/systemd/system-preset/80-ceph-installer.* -- gen_context(system_u:object_r:ceph_installer_unit_file_t,s0) 6 | 7 | /usr/lib/systemd/system/ceph-installer-celery.* -- gen_context(system_u:object_r:ceph_installer_unit_file_t,s0) 8 | 9 | /usr/lib/systemd/system/ceph-installer.* -- gen_context(system_u:object_r:ceph_installer_unit_file_t,s0) 10 | 11 | /var/lib/ceph-installer(/.*)? gen_context(system_u:object_r:ceph_installer_var_lib_t,s0) 12 | -------------------------------------------------------------------------------- /ceph_installer/cli/log.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is really not a logging facility. For the CLI we only require a nice color 3 | output on some interactions. Full logging is not needed. 4 | """ 5 | import sys 6 | from ceph_installer.cli import util 7 | 8 | 9 | def error(message): 10 | line = "%s %s\n" % (util.red_arrow, message) 11 | line = line.encode("utf-8") 12 | sys.stderr.write(line) 13 | 14 | 15 | def debug(message): 16 | line = "%s %s\n" % (util.blue_arrow, message) 17 | line = line.encode("utf-8") 18 | sys.stdout.write(line) 19 | 20 | 21 | def info(message): 22 | line = "%s %s\n" % (util.bold_arrow, message) 23 | line = line.encode("utf-8") 24 | sys.stdout.write(line) 25 | 26 | 27 | def warning(message): 28 | line = "%s %s\n" % (util.yellow_arrow, message) 29 | line = line.encode("utf-8") 30 | sys.stderr.write(line) 31 | -------------------------------------------------------------------------------- /ceph_installer/commands/populate.py: -------------------------------------------------------------------------------- 1 | from pecan.commands.base import BaseCommand 2 | from pecan import conf 3 | 4 | from ceph_installer import models 5 | 6 | 7 | def out(string): 8 | print "==> %s" % string 9 | 10 | 11 | class PopulateCommand(BaseCommand): 12 | """ 13 | Load a pecan environment and initializate the database. 14 | """ 15 | 16 | def run(self, args): 17 | super(PopulateCommand, self).run(args) 18 | out("LOADING ENVIRONMENT") 19 | self.load_app() 20 | out("BUILDING SCHEMA") 21 | try: 22 | out("STARTING A TRANSACTION...") 23 | models.start() 24 | models.Base.metadata.create_all(conf.sqlalchemy.engine) 25 | except: 26 | models.rollback() 27 | out("ROLLING BACK... ") 28 | raise 29 | else: 30 | out("COMMITING... ") 31 | models.commit() 32 | -------------------------------------------------------------------------------- /ceph_installer/controllers/tasks.py: -------------------------------------------------------------------------------- 1 | from pecan import expose 2 | 3 | from ceph_installer.models import Task 4 | from ceph_installer.controllers import error 5 | 6 | 7 | class TaskController(object): 8 | 9 | def __init__(self, task_id): 10 | self.task = Task.query.filter_by(identifier=task_id).first() 11 | if not self.task: 12 | error(404, '%s is not available' % task_id) 13 | 14 | @expose('json') 15 | def index(self): 16 | return self.task 17 | 18 | 19 | class TasksController(object): 20 | 21 | @expose('json') 22 | def index(self): 23 | return Task.query.all() 24 | 25 | @expose('json') 26 | def install(self): 27 | return {} 28 | 29 | @expose('json') 30 | def configure(self): 31 | return {} 32 | 33 | @expose() 34 | def _lookup(self, task_id, *remainder): 35 | return TaskController(task_id), remainder 36 | -------------------------------------------------------------------------------- /ceph_installer/controllers/root.py: -------------------------------------------------------------------------------- 1 | from pecan import expose 2 | from ceph_installer.controllers import ( 3 | tasks, mon, osd, rgw, calamari, errors, setup, agent, 4 | status 5 | ) 6 | 7 | 8 | class ApiController(object): 9 | 10 | @expose('json') 11 | def index(self): 12 | # TODO: allow some autodiscovery here so that clients can see what is 13 | # available 14 | return dict() 15 | 16 | agent = agent.AgentController() 17 | tasks = tasks.TasksController() 18 | mon = mon.MONController() 19 | osd = osd.OSDController() 20 | rgw = rgw.RGWController() 21 | calamari = calamari.CalamariController() 22 | status = status.StatusController() 23 | 24 | 25 | class RootController(object): 26 | 27 | @expose('json') 28 | def index(self): 29 | return dict() 30 | 31 | api = ApiController() 32 | errors = errors.ErrorController() 33 | setup = setup.SetupController() 34 | -------------------------------------------------------------------------------- /rpm/README.rst: -------------------------------------------------------------------------------- 1 | Building RPMs 2 | ============= 3 | Here are the steps to build an RPM from a Git snapshot (your current 4 | ``HEAD``), assuming you're on a CentOS 7 or RHEL 7 host:: 5 | 6 | # Enable EPEL 7 | sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm 8 | 9 | # Install "make" and "fedpkg": 10 | sudo yum -y install make fedpkg 11 | 12 | # Add your user account to the "mock" group. 13 | sudo usermod -a -G mock $(whoami) 14 | 15 | # Make the RPM snapshot: 16 | make rpm 17 | 18 | If the build fails, try checking ``root.log`` and ``build.log``, like so:: 19 | 20 | tail -n +1 {root,build}.log 21 | 22 | 23 | This "rpm" directory 24 | ==================== 25 | This directory contains the mock configuration file for ceph-installer to build 26 | in a mock chroot with all its build-time dependencies. 27 | 28 | For more general information about mock, see the project home page at 29 | https://github.com/rpm-software-management/mock/wiki 30 | -------------------------------------------------------------------------------- /ceph_installer/controllers/__init__.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | from pecan import request, redirect 3 | 4 | def error(url, msg=None): 5 | """ 6 | Helper for controller methods to do an internal redirect with 7 | either a url part like:: 8 | 9 | error("/errors/not_allowed/") 10 | 11 | Or an http code:: 12 | 13 | error(404) 14 | 15 | The ``msg`` argument is optional and would override the default error 16 | message for each error condition as defined in the ``ErrorController`` 17 | methods. 18 | """ 19 | code_to_url = { 20 | 400: '/errors/invalid/', 21 | 403: '/errors/forbidden/', 22 | 404: '/errors/not_found/', 23 | 405: '/errors/not_allowed/', 24 | 500: '/errors/error/', 25 | 503: '/errors/unavailable/', 26 | } 27 | 28 | if isinstance(url, int): 29 | url = code_to_url.get(url, 500) 30 | 31 | if msg: 32 | request.context['message'] = msg 33 | url = path.join(url, '?message=%s' % msg) 34 | redirect(url, internal=True) 35 | -------------------------------------------------------------------------------- /tests/functional/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = {ansible2.2}-{nightly_xenial,nightly_centos7} 3 | skipsdist = True 4 | 5 | [testenv] 6 | whitelist_externals = 7 | vagrant 8 | bash 9 | passenv=* 10 | setenv= 11 | ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config 12 | ansible2.2: ANSIBLE_STDOUT_CALLBACK = debug 13 | ANSIBLE_RETRY_FILES_ENABLED = False 14 | deps= 15 | ansible1.9: ansible==1.9.4 16 | ansible2.1: ansible==2.1 17 | ansible2.2: ansible==2.2.3 18 | -r{toxinidir}/requirements.txt 19 | changedir= 20 | nightly_xenial: {toxinidir}/nightly-xenial 21 | nightly_centos7: {toxinidir}/nightly-centos7 22 | commands= 23 | vagrant up --no-provision {posargs:--provider=virtualbox} 24 | bash {toxinidir}/scripts/generate_ssh_config.sh {changedir} 25 | 26 | ansible-playbook -vv -i {changedir}/hosts {toxinidir}/playbooks/setup.yml \ 27 | --extra-vars="installer_dev_branch={env:INSTALLER_DEV_BRANCH:master} ceph_ansible_dev_branch={env:CEPH_ANSIBLE_DEV_BRANCH:master}" 28 | ansible-playbook -vv -i {changedir}/hosts {changedir}/test.yml 29 | 30 | vagrant destroy --force 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2016 Red Hat Inc. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 17 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 18 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 19 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 20 | OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /ceph_installer/async.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pecan 3 | 4 | from celery import Celery 5 | from celery.signals import worker_init 6 | from ceph_installer import models 7 | 8 | 9 | @worker_init.connect 10 | def bootstrap_pecan(signal, sender): 11 | try: 12 | config_path = os.environ['PECAN_CONFIG'] 13 | except KeyError: 14 | here = os.path.abspath(os.path.dirname(__file__)) 15 | # XXX this will not hold true when installing as a binary 16 | config_path = os.path.abspath(os.path.join(here, '../config/config.py')) 17 | 18 | pecan.configuration.set_config(config_path, overwrite=True) 19 | # Once configuration is set we need to initialize the models so that we can connect 20 | # to the DB wth a configured mapper. 21 | models.init_model() 22 | 23 | 24 | app = Celery('ceph_installer.async', broker='amqp://guest@localhost//', include=['ceph_installer.tasks']) 25 | # the default value of CELERYD_CONCURRENCY will be set to the number 26 | # of CPU/cores on the host. This is a problem because we need to ensure 27 | # that there is only one celery worker running. 28 | app.conf.update(CELERYD_CONCURRENCY=1) 29 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for constructing RPMs. 2 | # Try "make" (for SRPMS) or "make rpm" 3 | 4 | NAME = ceph-installer 5 | VERSION := $(shell PYTHONPATH=. python -c \ 6 | 'import ceph_installer; print ceph_installer.__version__') 7 | COMMIT := $(shell git rev-parse HEAD) 8 | SHORTCOMMIT := $(shell echo $(COMMIT) | cut -c1-7) 9 | RELEASE := $(shell git describe --match 'v*' \ 10 | | sed 's/^v//' \ 11 | | sed 's/^[^-]*-//' \ 12 | | sed 's/-/./') 13 | ifeq ($(VERSION),$(RELEASE)) 14 | RELEASE = 0 15 | endif 16 | NVR := $(NAME)-$(VERSION)-$(RELEASE).el7 17 | 18 | all: srpm 19 | 20 | # Testing only 21 | echo: 22 | echo COMMIT $(COMMIT) 23 | echo VERSION $(VERSION) 24 | echo RELEASE $(RELEASE) 25 | echo NVR $(NVR) 26 | 27 | clean: 28 | rm -rf dist/ 29 | rm -rf ceph-installer-$(VERSION)-$(SHORTCOMMIT).tar.gz 30 | rm -rf $(NVR).src.rpm 31 | 32 | dist: 33 | python setup.py sdist \ 34 | && mv dist/ceph-installer-$(VERSION).tar.gz \ 35 | ceph-installer-$(VERSION)-$(SHORTCOMMIT).tar.gz 36 | 37 | spec: 38 | sed ceph-installer.spec.in \ 39 | -e 's/@COMMIT@/$(COMMIT)/' \ 40 | -e 's/@VERSION@/$(VERSION)/' \ 41 | -e 's/@RELEASE@/$(RELEASE)/' \ 42 | > ceph-installer.spec 43 | 44 | srpm: dist spec 45 | fedpkg --dist epel7 srpm 46 | 47 | rpm: dist srpm 48 | mock -r rpm/ktdreyer-ceph-installer.cfg rebuild $(NVR).src.rpm --resultdir=. 49 | 50 | .PHONY: dist rpm srpm 51 | -------------------------------------------------------------------------------- /deploy/playbooks/roles/common/tasks/systemd.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: ensure {{ app_home }}/log exists 4 | file: 5 | path: "{{ app_home }}/log" 6 | state: directory 7 | 8 | - name: ensure /etc/sysconfig/ dir exists 9 | sudo: true 10 | file: path=/etc/sysconfig state=directory 11 | 12 | # prevents issues when updating systemd files 13 | - name: reload systemd 14 | sudo: yes 15 | command: systemctl daemon-reload 16 | 17 | - name: install the systemd configuration file for celery 18 | template: 19 | src: ceph-installer.sysconfig.j2 20 | dest: /etc/sysconfig/ceph-installer 21 | sudo: true 22 | notify: 23 | - reload systemd 24 | 25 | - name: install the systemd unit file for ceph-installer 26 | template: 27 | src: ceph-installer.service.j2 28 | dest: /etc/systemd/system/ceph-installer.service 29 | sudo: true 30 | notify: 31 | - reload systemd 32 | 33 | - name: install the systemd unit file for celery 34 | template: 35 | src: ceph-installer-celery.service.j2 36 | dest: /etc/systemd/system/ceph-installer-celery.service 37 | sudo: true 38 | notify: 39 | - reload systemd 40 | 41 | - name: ensure ceph-installer-celery is enabled and running 42 | sudo: true 43 | service: 44 | name: ceph-installer-celery 45 | state: running enabled=yes 46 | 47 | - name: ensure ceph-installer is enabled and running 48 | sudo: true 49 | service: 50 | name: ceph-installer 51 | state: running enabled=yes 52 | -------------------------------------------------------------------------------- /ceph_installer/tests/controllers/test_status.py: -------------------------------------------------------------------------------- 1 | from ceph_installer import hooks 2 | from ceph_installer.controllers import status 3 | 4 | 5 | def generic_system_error(): 6 | msg = "important system is not running" 7 | raise hooks.SystemCheckError(msg) 8 | 9 | error_checks = ( 10 | generic_system_error, 11 | ) 12 | 13 | ok_checks = ( 14 | lambda: True, 15 | ) 16 | 17 | 18 | class TestSetupController(object): 19 | 20 | def test_index_system_error_message(self, session, monkeypatch): 21 | monkeypatch.setattr(status, 'system_checks', error_checks) 22 | result = session.app.get("/api/status/", expect_errors=True) 23 | assert result.json['message'] == 'important system is not running' 24 | 25 | def test_index_system_error_code(self, session, monkeypatch): 26 | monkeypatch.setattr(status, 'system_checks', error_checks) 27 | result = session.app.get("/api/status/", expect_errors=True) 28 | assert result.status_int == 500 29 | 30 | def test_index_system_ok_message(self, session, monkeypatch): 31 | monkeypatch.setattr(status, 'system_checks', ok_checks) 32 | result = session.app.get("/api/status/") 33 | assert result.json['message'] == "ok" 34 | 35 | def test_index_system_ok_status(self, session, monkeypatch): 36 | monkeypatch.setattr(status, 'system_checks', ok_checks) 37 | result = session.app.get("/api/status/") 38 | assert result.status_int == 200 39 | -------------------------------------------------------------------------------- /ceph_installer/app.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pecan import make_app 3 | from ceph_installer import models, process 4 | from ceph_installer.util import mkdir 5 | 6 | 7 | def ensure_ssh_keys(): 8 | """ 9 | Generate ssh keys as early as possible so that they are available to all 10 | web server workers immediately when serving ``/setup/key/``. This helper 11 | does not use logging because it is too early in running the application 12 | and no logging has been configured yet. 13 | """ 14 | # look for the ssh key of the current user 15 | private_key_path = os.path.expanduser('~/.ssh/id_rsa') 16 | public_key_path = os.path.expanduser('~/.ssh/id_rsa.pub') 17 | ssh_dir = os.path.dirname(public_key_path) 18 | 19 | if not os.path.isdir(ssh_dir): 20 | mkdir(ssh_dir) 21 | 22 | # if there isn't one create it 23 | if not os.path.exists(public_key_path): 24 | # create one 25 | command = [ 26 | 'ssh-keygen', '-q', '-t', 'rsa', 27 | '-N', '', 28 | '-f', private_key_path, 29 | ] 30 | out, err, code = process.run(command, send_input='y\n') 31 | if code != 0: 32 | raise RuntimeError('ssh-keygen failed: %s %s' % (out, err)) 33 | 34 | 35 | def setup_app(config): 36 | ensure_ssh_keys() 37 | models.init_model() 38 | app_conf = dict(config.app) 39 | 40 | return make_app( 41 | app_conf.pop('root'), 42 | logging=getattr(config, 'logging', {}), 43 | **app_conf 44 | ) 45 | -------------------------------------------------------------------------------- /ceph_installer/tests/controllers/test_setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ceph_installer.controllers import setup 3 | 4 | 5 | class TestSetupController(object): 6 | 7 | def test_index_generates_a_script(self, session): 8 | result = session.app.get( 9 | '/setup/', 10 | extra_environ=dict(REMOTE_ADDR='192.168.1.1') 11 | ) 12 | assert '#!/bin/bash' in result.body 13 | 14 | def test_index_works_for_remote_requests(self, session): 15 | result = session.app.get('/setup/') 16 | assert '#!/bin/bash' in result.body 17 | 18 | def test_index_adds_the_right_endpoint_to_the_script(self, session): 19 | result = session.app.get('/setup/') 20 | assert 'http://localhost/setup/key/' in result.body 21 | 22 | def test_missing_ssh_directory(self, session, tmpdir, monkeypatch): 23 | rsa_path = os.path.join(str(tmpdir), '.ssh/id_rsa') 24 | monkeypatch.setattr(setup.os.path, 'expanduser', lambda x: rsa_path) 25 | result = session.app.get('/setup/key/', expect_errors=True) 26 | assert result.status_int == 500 27 | assert result.json['message'].startswith('.ssh directory not found') 28 | 29 | def test_missing_ssh_key(self, session, tmpdir, monkeypatch): 30 | tmpdir.mkdir('.ssh') 31 | rsa_path = os.path.join(str(tmpdir), '.ssh/id_rsa') 32 | monkeypatch.setattr(setup.os.path, 'expanduser', lambda x: rsa_path) 33 | result = session.app.get('/setup/key/', expect_errors=True) 34 | assert result.status_int == 500 35 | assert result.json['message'].startswith('expected public key not found') 36 | -------------------------------------------------------------------------------- /ceph_installer/controllers/agent.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pecan import expose, request 4 | from pecan.ext.notario import validate 5 | from uuid import uuid4 6 | 7 | from ceph_installer.controllers import error 8 | from ceph_installer.tasks import call_ansible 9 | from ceph_installer import schemas 10 | from ceph_installer import models 11 | from ceph_installer import util 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | class AgentController(object): 18 | 19 | @expose(generic=True, template='json') 20 | def index(self): 21 | error(405) 22 | 23 | @index.when(method='POST', template='json') 24 | @validate(schemas.agent_install_schema, handler="/errors/schema") 25 | def install(self): 26 | master = request.json.get('master', request.server_name) 27 | logger.info('defining "%s" as the master host for the minion configuration', master) 28 | hosts = request.json.get('hosts') 29 | verbose_ansible = request.json.get('verbose', False) 30 | extra_vars = util.get_install_extra_vars(request.json) 31 | extra_vars['agent_master_host'] = master 32 | identifier = str(uuid4()) 33 | task = models.Task( 34 | identifier=identifier, 35 | endpoint=request.path, 36 | ) 37 | # we need an explicit commit here because the command may finish before 38 | # we conclude this request 39 | models.commit() 40 | kwargs = dict(extra_vars=extra_vars, verbose=verbose_ansible) 41 | call_ansible.apply_async( 42 | args=([('agents', hosts)], identifier), 43 | kwargs=kwargs, 44 | ) 45 | 46 | return task 47 | -------------------------------------------------------------------------------- /ceph_installer/tests/config.py: -------------------------------------------------------------------------------- 1 | from ceph_installer.hooks import LocalHostWritesHook 2 | # Server Specific Configurations 3 | server = { 4 | 'port': '8181', 5 | 'host': '0.0.0.0' 6 | } 7 | 8 | # Pecan Application Configurations 9 | app = { 10 | 'root': 'ceph_installer.controllers.root.RootController', 11 | 'modules': ['ceph_installer'], 12 | 'debug': False, 13 | 'hooks': [LocalHostWritesHook()], 14 | } 15 | 16 | logging = { 17 | 'root': {'level': 'INFO', 'handlers': ['console']}, 18 | 'loggers': { 19 | 'ceph_installer': {'level': 'DEBUG', 'handlers': ['console'], 'propagate': False}, 20 | 'pecan': {'level': 'INFO', 'handlers': ['console'], 'propagate': False}, 21 | 'py.warnings': {'handlers': ['console']}, 22 | '__force_dict__': True 23 | }, 24 | # XXX Determine the right location for logs 25 | 'handlers': { 26 | 'console': { 27 | 'level': 'DEBUG', 28 | 'class': 'logging.StreamHandler', 29 | 'formatter': 'color' 30 | } 31 | }, 32 | 'formatters': { 33 | 'simple': { 34 | 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' 35 | '[%(threadName)s] %(message)s') 36 | }, 37 | 'color': { 38 | '()': 'pecan.log.ColorFormatter', 39 | 'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]' 40 | '[%(threadName)s] %(message)s'), 41 | '__force_dict__': True 42 | } 43 | } 44 | } 45 | 46 | 47 | sqlalchemy = { 48 | # XXX Determine the right location for the database 49 | 'url': 'sqlite:////tmp/ceph_installertest.db', 50 | 'echo': True, 51 | 'echo_pool': True, 52 | 'pool_recycle': 3600, 53 | 'encoding': 'utf-8' 54 | } 55 | -------------------------------------------------------------------------------- /config/config.py: -------------------------------------------------------------------------------- 1 | from ceph_installer import hooks 2 | 3 | # Server Specific Configurations 4 | server = { 5 | 'port': '8181', 6 | 'host': '0.0.0.0' 7 | } 8 | 9 | # Pecan Application Configurations 10 | app = { 11 | 'root': 'ceph_installer.controllers.root.RootController', 12 | 'modules': ['ceph_installer'], 13 | 'debug': False, 14 | 'hooks': [hooks.CustomErrorHook(), hooks.LocalHostWritesHook()] 15 | } 16 | 17 | logging = { 18 | 'root': {'level': 'INFO', 'handlers': ['console']}, 19 | 'loggers': { 20 | 'ceph_installer': {'level': 'DEBUG', 'handlers': ['console'], 'propagate': False}, 21 | 'pecan': {'level': 'INFO', 'handlers': ['console'], 'propagate': False}, 22 | 'py.warnings': {'handlers': ['console']}, 23 | '__force_dict__': True 24 | }, 25 | # XXX Determine the right location for logs 26 | 'handlers': { 27 | 'console': { 28 | 'level': 'DEBUG', 29 | 'class': 'logging.StreamHandler', 30 | 'formatter': 'simple' 31 | } 32 | }, 33 | 'formatters': { 34 | 'simple': { 35 | 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' 36 | '[%(threadName)s] %(message)s') 37 | }, 38 | 'color': { 39 | '()': 'pecan.log.ColorFormatter', 40 | 'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]' 41 | '[%(threadName)s] %(message)s'), 42 | '__force_dict__': True 43 | } 44 | } 45 | } 46 | 47 | 48 | sqlalchemy = { 49 | # XXX Determine the right location for the database 50 | 'url': 'sqlite:////var/lib/ceph-installer/ceph_installer.db', 51 | 'echo': True, 52 | 'echo_pool': True, 53 | 'pool_recycle': 3600, 54 | 'encoding': 'utf-8' 55 | } 56 | -------------------------------------------------------------------------------- /ceph_installer/cli/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from tambo import Transport 4 | import ceph_installer 5 | from ceph_installer.cli import log 6 | from ceph_installer.cli import dev, task, constants 7 | from ceph_installer.cli.decorators import catches 8 | 9 | 10 | class CephInstaller(object): 11 | _help = """ 12 | A command line utility to install and configure Ceph using an HTTP API as a REST service 13 | to call Ansible. 14 | 15 | Address: %s 16 | Version: %s 17 | 18 | Global Options: 19 | -h, --help, help Show this program's help menu 20 | --log, --logging Set the level of logging. Acceptable values: 21 | debug, warning, error, critical 22 | 23 | Environment Variables: 24 | CEPH_INSTALLER_ADDRESS Define the location of the installer. 25 | Defaults to "http://localhost:8181" 26 | 27 | %s 28 | """ 29 | 30 | mapper = {'dev': dev.Dev, 'task': task.Task} 31 | 32 | def __init__(self, argv=None, parse=True): 33 | self.plugin_help = "No plugins found/loaded" 34 | if argv is None: 35 | argv = sys.argv 36 | if parse: 37 | self.main(argv) 38 | 39 | def help(self, subhelp): 40 | version = ceph_installer.__version__ 41 | return self._help % ( 42 | constants.server_address, version, 43 | subhelp 44 | ) 45 | 46 | @catches(KeyboardInterrupt, handle_all=True, logger=log) 47 | def main(self, argv): 48 | parser = Transport(argv, mapper=self.mapper, 49 | options=[], check_help=False, 50 | check_version=False) 51 | parser.parse_args() 52 | parser.catch_help = self.help(parser.subhelp()) 53 | parser.catch_version = ceph_installer.__version__ 54 | parser.mapper = self.mapper 55 | if len(argv) <= 1: 56 | return parser.print_help() 57 | parser.dispatch() 58 | parser.catches_help() 59 | parser.catches_version() 60 | -------------------------------------------------------------------------------- /ceph_installer/models/tasks.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, Integer, String, Boolean, DateTime, UnicodeText 2 | from sqlalchemy.orm.exc import DetachedInstanceError 3 | from ceph_installer.models import Base 4 | 5 | 6 | class Task(Base): 7 | 8 | __tablename__ = 'tasks' 9 | id = Column(Integer, primary_key=True) 10 | identifier = Column(String(256), unique=True, nullable=False, index=True) 11 | endpoint = Column(String(256), index=True) 12 | user_agent = Column(String(512)) 13 | request = Column(UnicodeText) 14 | http_method = Column(String(64)) 15 | command = Column(String(256)) 16 | stderr = Column(UnicodeText) 17 | stdout = Column(UnicodeText) 18 | started = Column(DateTime) 19 | ended = Column(DateTime) 20 | succeeded = Column(Boolean(), default=False) 21 | exit_code = Column(Integer) 22 | 23 | def __init__(self, request=None, **kw): 24 | self._extract_request_metadata(request) 25 | for k, v in kw.items(): 26 | setattr(self, k, v) 27 | 28 | def _extract_request_metadata(self, request): 29 | self.http_method = getattr(request, 'method', '') 30 | self.request = str(getattr(request, 'body', '')) 31 | self.user_agent = getattr(request, 'user_agent', '') 32 | 33 | def __repr__(self): 34 | try: 35 | return '' % self.identifier 36 | except DetachedInstanceError: 37 | return '' 38 | 39 | def __json__(self): 40 | return dict( 41 | identifier = self.identifier, 42 | endpoint = self.endpoint, 43 | command = self.command, 44 | stderr = self.stderr, 45 | stdout = self.stdout, 46 | started = self.started, 47 | ended = self.ended, 48 | succeeded = self.succeeded, 49 | exit_code = self.exit_code, 50 | user_agent = self.user_agent, 51 | request = self.request, 52 | http_method = self.http_method, 53 | ) 54 | -------------------------------------------------------------------------------- /ceph_installer/templates.py: -------------------------------------------------------------------------------- 1 | 2 | setup_script = """#!/bin/bash 3 | if [[ $EUID -ne 0 ]]; then 4 | echo "You must be a root user or execute this script with sudo" 2>&1 5 | exit 1 6 | fi 7 | 8 | if [ ! -f /etc/os-release ]; then 9 | echo "/etc/os-release is not found. This system is not supported." 10 | echo "will not proceed with installation" 11 | exit 2 12 | fi 13 | 14 | source /etc/os-release 15 | if [ "$ID" != "ubuntu" ] && [ "$ID" != "rhel" ] && [ "$ID" != "centos" ]; then 16 | echo "Unsupported system detected: $ID" 17 | echo "will not proceed with installation" 18 | exit 3 19 | fi 20 | 21 | 22 | if [ "$ID" == "ubuntu" ]; then 23 | echo "--> Installing Python 2.7 for Ansible" 24 | apt-get update 25 | DEBIAN_FRONTEND=noninteractive apt-get -y install python 26 | fi 27 | 28 | echo "--> creating new user with disabled password: ceph-installer" 29 | useradd -m ceph-installer 30 | passwd -d ceph-installer 31 | 32 | echo "--> adding provisioning key to the ceph-installer user authorized_keys" 33 | curl -s -L -o ansible.pub {ssh_key_address} 34 | mkdir -m 700 -p /home/ceph-installer/.ssh 35 | cat ansible.pub >> /home/ceph-installer/.ssh/authorized_keys 36 | 37 | echo "--> ensuring correct permissions on .ssh/authorized_keys" 38 | chown -R ceph-installer:ceph-installer /home/ceph-installer/.ssh 39 | chmod 600 /home/ceph-installer/.ssh/authorized_keys 40 | 41 | echo "--> ensuring that ceph-installer user will be able to sudo" 42 | # write to it wiping everything 43 | echo "ceph-installer ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/ceph-installer 44 | 45 | echo "--> ensuring ceph-installer user does not require a tty" 46 | # and now just append 47 | echo 'Defaults:ceph-installer !requiretty' >> /etc/sudoers.d/ceph-installer 48 | """ 49 | 50 | # Note that the agent_script can't run by itself, it needs to be concatenated 51 | # along the regular setup script 52 | agent_script = """ 53 | echo "--> installing and configuring agent" 54 | curl -d '{{"hosts": ["{target_host}"]}}' -X POST {agent_endpoint} 55 | """ 56 | -------------------------------------------------------------------------------- /ceph_installer/controllers/setup.py: -------------------------------------------------------------------------------- 1 | from pecan import expose, request, response 2 | from webob.static import FileIter 3 | from ceph_installer.util import make_setup_script, make_agent_script 4 | from ceph_installer.controllers import error 5 | import os 6 | from StringIO import StringIO 7 | import logging 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class SetupController(object): 13 | 14 | @expose(content_type='application/octet-stream') 15 | def index(self): 16 | script = make_setup_script(request.url) 17 | response.headers['Content-Disposition'] = 'attachment; filename=setup.sh' 18 | response.app_iter = FileIter(script) 19 | 20 | @expose(content_type='application/octet-stream') 21 | def agent(self): 22 | script = make_agent_script(request.url, request.client_addr) 23 | response.headers['Content-Disposition'] = 'attachment; filename=agent-setup.sh' 24 | response.app_iter = FileIter(script) 25 | 26 | @expose(content_type='application/octet-stream') 27 | def key(self): 28 | """ 29 | Serves the public SSH key for the user that own the current service 30 | """ 31 | # look for the ssh key of the current user 32 | public_key_path = os.path.expanduser('~/.ssh/id_rsa.pub') 33 | ssh_dir = os.path.dirname(public_key_path) 34 | 35 | if not os.path.isdir(ssh_dir): 36 | msg = '.ssh directory not found: %s' % ssh_dir 37 | logger.error(msg) 38 | error(500, msg) 39 | 40 | if not os.path.exists(public_key_path): 41 | msg = 'expected public key not found: %s' % public_key_path 42 | logger.error(msg) 43 | error(500, msg) 44 | 45 | # define the file to download 46 | response.headers['Content-Disposition'] = 'attachment; filename=id_rsa.pub' 47 | with open(public_key_path) as key_contents: 48 | key = StringIO() 49 | key.write(key_contents.read()) 50 | key.seek(0) 51 | response.app_iter = FileIter(key) 52 | -------------------------------------------------------------------------------- /ceph_installer/tests/controllers/test_rgw.py: -------------------------------------------------------------------------------- 1 | from ceph_installer.controllers import rgw 2 | 3 | 4 | class TestRGWController(object): 5 | 6 | def setup(self): 7 | data = dict( 8 | host="node1", 9 | fsid="1720107309134", 10 | monitors=[{"host": "mon1.host", "address": "10.0.0.1"}], 11 | public_network="0.0.0.0/24", 12 | ) 13 | self.configure_data = data 14 | 15 | def test_index_get(self, session): 16 | result = session.app.get("/api/rgw/") 17 | assert result.status_int == 200 18 | 19 | def test_install_hosts(self, session, monkeypatch): 20 | monkeypatch.setattr(rgw.call_ansible, 'apply_async', lambda args, kwargs: None) 21 | data = dict(hosts=["node1"]) 22 | result = session.app.post_json("/api/rgw/install/", params=data) 23 | assert result.json['endpoint'] == '/api/rgw/install/' 24 | assert result.json['identifier'] is not None 25 | 26 | def test_install_missing_hosts(self, session): 27 | result = session.app.post_json("/api/rgw/install/", params=dict(), 28 | expect_errors=True) 29 | assert result.status_int == 400 30 | 31 | def test_install_bogus_field(self, session): 32 | data = dict(hosts=["google.com"], bogus="foo") 33 | result = session.app.post_json("/api/rgw/install/", params=data, 34 | expect_errors=True) 35 | assert result.status_int == 400 36 | 37 | def test_configure_missing_fields(self, session): 38 | data = dict() 39 | result = session.app.post_json("/api/rgw/configure/", params=data, 40 | expect_errors=True) 41 | assert result.status_int == 400 42 | 43 | def test_configure_success(self, session, monkeypatch): 44 | monkeypatch.setattr(rgw.call_ansible, 'apply_async', lambda args, kwargs: None) 45 | result = session.app.post_json("/api/rgw/configure/", params=self.configure_data) 46 | assert result.json['endpoint'] == '/api/rgw/configure/' 47 | assert result.json['identifier'] is not None 48 | -------------------------------------------------------------------------------- /deploy/playbooks/roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: "ensure a home for {{ app_name }}" 4 | sudo: yes 5 | file: 6 | path: "{{ app_home }}" 7 | owner: "{{ ansible_ssh_user }}" 8 | group: "{{ ansible_ssh_user }}" 9 | state: directory 10 | recurse: yes 11 | register: app_home_created 12 | 13 | - name: "create /var/lib database location for {{ app_name }}" 14 | sudo: yes 15 | file: 16 | path: "/var/lib/{{ app_name }}" 17 | owner: "{{ ansible_ssh_user }}" 18 | group: "{{ ansible_ssh_user }}" 19 | state: directory 20 | recurse: yes 21 | sudo: yes 22 | 23 | - name: install EPEL 24 | sudo: yes 25 | package: 26 | name: epel-release 27 | state: present 28 | tags: 29 | - packages 30 | 31 | - name: install ssl system requirements 32 | sudo: yes 33 | package: 34 | name: "{{ item }}" 35 | state: present 36 | with_items: "{{ ssl_requirements }}" 37 | when: app_use_ssl 38 | tags: 39 | - packages 40 | 41 | - name: install system packages 42 | sudo: yes 43 | package: 44 | name: "{{ item }}" 45 | state: present 46 | with_items: "{{ system_packages }}" 47 | tags: 48 | - packages 49 | 50 | - name: pip install setuptools 51 | sudo: yes 52 | pip: 53 | name: setuptools 54 | 55 | - name: pip install setuptools, pip, and virtualenv 56 | sudo: yes 57 | pip: 58 | name: "{{ item }}" 59 | extra_args: '--upgrade' 60 | with_items: 61 | - pip 62 | - setuptools 63 | - virtualenv 64 | 65 | - name: "pip+git install {{ app_name }} into virtualenv." 66 | pip: 67 | name: 'git+https://github.com/ceph/ceph-installer@{{ branch }}#egg=ceph_installer' 68 | virtualenv: "{{ app_home }}" 69 | changed_when: True 70 | 71 | - name: populate the database for {{ app_name }} 72 | command: "{{ app_home }}/bin/pecan populate {{ app_home }}/src/{{ app_name }}/config/config.py" 73 | 74 | - name: clone ceph-ansible to {{ app_home }} 75 | git: 76 | repo: "https://github.com/ceph/ceph-ansible.git" 77 | dest: "{{ app_home }}/ceph-ansible" 78 | force: yes 79 | update: yes 80 | 81 | - include: systemd.yml 82 | tags: 83 | - systemd 84 | -------------------------------------------------------------------------------- /docs/source/man/index.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | ceph-installer 3 | ============== 4 | 5 | ------------------------------------------------------------------------- 6 | Command line utility to install and configure Ceph using an HTTP REST API 7 | ------------------------------------------------------------------------- 8 | 9 | :Manual section: 8 10 | 11 | Global Options 12 | -------------- 13 | 14 | -h, --help, help Show this program's help menu 15 | 16 | --log, --logging Set the level of logging. Acceptable values: debug, warning, error, critical 17 | 18 | Environment Variables 19 | --------------------- 20 | 21 | CEPH_INSTALLER_ADDRESS 22 | Define the location of the installer. 23 | 24 | Defaults to "http://localhost:8181" 25 | 26 | Commands 27 | -------- 28 | 29 | task 30 | ++++ 31 | 32 | 33 | Human-readable task information: stdout, stderr, and the ability to "poll" 34 | a task that waits until the command completes to be able to show the output 35 | in a readable way. 36 | 37 | Usage:: 38 | 39 | ceph-installer task $IDENTIFIER 40 | 41 | Options:: 42 | 43 | --poll Poll until the task has completed (either on failure or success) 44 | stdout Retrieve the stdout output from the task 45 | stderr Retrieve the stderr output from the task 46 | command The actual command used to call ansible 47 | ended The timestamp (in UTC) when the command completed 48 | started The timestamp (in UTC) when the command started 49 | exit_code The shell exit status for the process 50 | succeeded Boolean value to indicate if process completed correctly 51 | 52 | dev 53 | +++ 54 | 55 | 56 | Deploying the ceph-installer HTTP service to a remote server with ansible. 57 | This command wraps ansible and certain flags to make it easier to deploy 58 | a development version. 59 | 60 | Usage:: 61 | 62 | ceph-installer dev $HOST 63 | 64 | Note: Requires a remote user with passwordless sudo. User defaults to 65 | "vagrant". 66 | 67 | Options: 68 | 69 | --user Define a user to connect to the remote server. Defaults to 'vagrant' 70 | --branch What branch to use for the deployment. Defaults to 'master' 71 | -vvvv Enable high verbosity when running ansible 72 | 73 | -------------------------------------------------------------------------------- /ceph_installer/cli/dev.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | from textwrap import dedent 3 | from tambo import Transport 4 | 5 | from ceph_installer import process 6 | from ceph_installer.cli import log 7 | 8 | this_dir = path.abspath(path.dirname(__file__)) 9 | top_dir = path.dirname(path.dirname(this_dir)) 10 | playbook_path = path.join(top_dir, 'deploy/playbooks') 11 | 12 | 13 | class Dev(object): 14 | 15 | help = "Development options" 16 | options = ['--user', '--branch'] 17 | _help = dedent(""" 18 | Deploying the ceph-installer HTTP service to a remote server with ansible. 19 | This command wraps ansible and certain flags to make it easier to deploy 20 | a development version. 21 | 22 | Usage:: 23 | 24 | ceph-installer dev $HOST 25 | 26 | Note: Requires a remote user with passwordless sudo. User defaults to 27 | "vagrant". 28 | 29 | Options: 30 | 31 | --user Define a user to connect to the remote server. Defaults to 'vagrant' 32 | --branch What branch to use for the deployment. Defaults to 'master' 33 | -vvvv Enable high verbosity when running ansible 34 | """) 35 | 36 | def __init__(self, arguments): 37 | self.arguments = arguments 38 | 39 | def main(self): 40 | parser = Transport(self.arguments, options=self.options, check_help=True) 41 | parser.catch_help = self._help 42 | parser.parse_args() 43 | parser.catches_help() 44 | branch = parser.get('--branch', 'master') 45 | user = parser.get('--user', 'vagrant') 46 | high_verbosity = '-vvvv' if parser.has('-vvvv') else '-v' 47 | if not parser.unknown_commands: 48 | log.error("it is required to pass a host to deploy to, but none was provided") 49 | raise SystemExit(1) 50 | 51 | command = [ 52 | "ansible-playbook", 53 | "-i", "%s," % parser.unknown_commands[-1], 54 | high_verbosity, 55 | "-u", user, 56 | "--extra-vars", 'branch=%s' % branch, 57 | "deploy.yml", 58 | ] 59 | log.debug("Running command: %s" % ' '.join(command)) 60 | out, err, code = process.run(command, cwd=playbook_path) 61 | log.error(err) 62 | log.debug(out) 63 | -------------------------------------------------------------------------------- /ceph_installer/controllers/errors.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pecan import expose, response, request 3 | 4 | 5 | logger = logging.getLogger(__name__) 6 | schema_logger = logging.getLogger("%s.schema" % __name__) 7 | 8 | 9 | class ErrorController(object): 10 | 11 | @expose('json') 12 | def schema(self, **kw): 13 | response.status = 400 14 | schema_logger.error(request.validation_error) 15 | try: 16 | path = request.validation_error._format_path() 17 | message = '%s%sfailed validation, %s' % ( 18 | path, 19 | '' if path.endswith(' ') else ' ', 20 | request.validation_error.reason 21 | ) 22 | except AttributeError: 23 | message = "invalid JSON was received" 24 | return dict(message=message) 25 | 26 | @expose('json') 27 | def invalid(self, **kw): 28 | msg = kw.get( 29 | 'message', 30 | 'invalid request' 31 | ) 32 | response.status = 400 33 | return dict(message=msg) 34 | 35 | @expose('json') 36 | def forbidden(self, **kw): 37 | msg = kw.get( 38 | 'message', 39 | 'forbidden' 40 | ) 41 | response.status = 403 42 | return dict(message=msg) 43 | 44 | @expose('json') 45 | def not_found(self, **kw): 46 | msg = kw.get( 47 | 'message', 48 | 'resource was not found' 49 | ) 50 | response.status = 404 51 | return dict(message=msg) 52 | 53 | @expose('json') 54 | def not_allowed(self, **kw): 55 | msg = kw.get( 56 | 'message', 57 | 'method %s not allowed for "%s"' % (request.method, request.path) 58 | ) 59 | response.status = 405 60 | return dict(message=msg) 61 | 62 | @expose('json') 63 | def unavailable(self, **kw): 64 | msg = kw.get( 65 | 'message', 66 | 'service unavailable', 67 | ) 68 | response.status = 503 69 | return dict(message=msg) 70 | 71 | @expose('json') 72 | def error(self, **kw): 73 | msg = kw.get( 74 | 'message', 75 | 'an error has occured', 76 | ) 77 | response.status = 500 78 | return dict(message=msg) 79 | -------------------------------------------------------------------------------- /ceph_installer/cli/util.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | class colorize(str): 5 | """ 6 | Pretty simple to use:: 7 | 8 | colorize.make('foo').bold 9 | colorize.make('foo').green 10 | colorize.make('foo').yellow 11 | colorize.make('foo').red 12 | colorize.make('foo').blue 13 | 14 | Otherwise you could go the long way (for example if you are 15 | testing this class):: 16 | 17 | string = colorize('foo') 18 | string._set_attributes() 19 | string.red 20 | 21 | """ 22 | 23 | def __init__(self, string): 24 | self.stdout = sys.__stdout__ 25 | self.appends = '' 26 | self.prepends = '' 27 | self.isatty = self.stdout.isatty() 28 | 29 | def _set_attributes(self): 30 | """ 31 | Sets the attributes here because the str class does not 32 | allow to pass in anything other than a string to the constructor 33 | so we can't really mess with the other attributes. 34 | """ 35 | for k, v in self.__colors__.items(): 36 | setattr(self, k, self.make_color(v)) 37 | 38 | def make_color(self, color): 39 | if not self.isatty or self.is_windows: 40 | return self 41 | return color + self + '\033[0m' + self.appends 42 | 43 | @property 44 | def __colors__(self): 45 | return dict( 46 | blue = '\033[34m', 47 | green = '\033[92m', 48 | yellow = '\033[33m', 49 | red = '\033[91m', 50 | bold = '\033[1m', 51 | ends = '\033[0m' 52 | ) 53 | 54 | @property 55 | def is_windows(self): 56 | if sys.platform == 'win32': 57 | return True 58 | return False 59 | 60 | @classmethod 61 | def make(cls, string): 62 | """ 63 | A helper method to return itself and workaround the fact that 64 | the str object doesn't allow extra arguments passed in to the 65 | constructor 66 | """ 67 | obj = cls(string) 68 | obj._set_attributes() 69 | return obj 70 | 71 | # 72 | # Common string manipulations 73 | # 74 | red_arrow = colorize.make('-->').red 75 | blue_arrow = colorize.make('-->').blue 76 | bold_arrow = colorize.make('-->').bold 77 | yellow_arrow = colorize.make('-->').yellow 78 | -------------------------------------------------------------------------------- /ceph_installer/tests/test_process.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from ceph_installer import process 4 | 5 | 6 | class TestMakeAnsibleCommand(object): 7 | 8 | def test_with_tags(self, monkeypatch): 9 | monkeypatch.setattr(process, 'which', lambda x: "/bin/ansible") 10 | result = process.make_ansible_command("/hosts", "uuid", tags="package-install") 11 | assert "--tags" in result 12 | assert "package-install" in result 13 | 14 | def test_with_skip_tags(self, monkeypatch): 15 | monkeypatch.setattr(process, 'which', lambda x: "/bin/ansible") 16 | result = process.make_ansible_command("/hosts", "uuid", skip_tags="package-install") 17 | assert "--skip-tags" in result 18 | assert "package-install" in result 19 | 20 | def test_with_extra_vars(self, monkeypatch): 21 | monkeypatch.setattr(process, 'which', lambda x: "/bin/ansible") 22 | result = process.make_ansible_command("/hosts", "uuid", extra_vars=dict(foo="bar")) 23 | assert '{"foo": "bar"}' in result 24 | assert "--extra-vars" in result 25 | 26 | def test_with_playbook(self, monkeypatch): 27 | monkeypatch.setattr(process, 'which', lambda x: "/bin/ansible") 28 | result = process.make_ansible_command("/hosts", "uuid", playbook="playbook.yml") 29 | assert "/usr/share/ceph-ansible/playbook.yml" in result 30 | 31 | 32 | class FakePopen(object): 33 | 34 | def __init__(self, stdout='', stderr='', returncode=0): 35 | self.stdout = stdout 36 | self.stderr = stderr 37 | self.returncode = returncode 38 | 39 | def communicate(self, *a): 40 | return self.stdout, self.stderr 41 | 42 | 43 | class TestProcess(object): 44 | 45 | def test_decode_unicode_on_the_fly_for_stdout(self, monkeypatch): 46 | monkeypatch.setattr( 47 | process.subprocess, 'Popen', 48 | lambda *a, **kw: FakePopen('£', 'stderr') 49 | ) 50 | stdout, stderr, code = process.run('ls') 51 | assert stdout == u'\xa3' 52 | 53 | def test_decode_unicode_on_the_fly_for_stderr(self, monkeypatch): 54 | monkeypatch.setattr( 55 | process.subprocess, 'Popen', 56 | lambda *a, **kw: FakePopen('stdout', '™') 57 | ) 58 | stdout, stderr, code = process.run('ls') 59 | assert stderr == u'\u2122' 60 | -------------------------------------------------------------------------------- /ceph_installer/tests/controllers/test_errors.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class TestSchemaErrors(object): 4 | 5 | def test_install_empty_object(self, session): 6 | result = session.app.post_json("/api/mon/install/", params=dict(), 7 | expect_errors=True) 8 | message = result.json['message'] 9 | assert message.endswith('an empty dictionary object was provided') 10 | 11 | def test_install_invalid_json(self, session): 12 | # note how we are not using post_json for the request 13 | result = session.app.post("/api/mon/install/", params={2: 1}, expect_errors=True) 14 | message = result.json['message'] 15 | assert message == 'invalid JSON was received' 16 | 17 | def test_host_is_invalid(self, session): 18 | params = {'hosts': ''} 19 | result = session.app.post_json("/api/mon/install/", params=params, 20 | expect_errors=True) 21 | message = result.json['message'] 22 | assert "hosts" in message 23 | assert message.endswith( 24 | "failed validation, requires format: ['host1', 'host2']" 25 | ) 26 | 27 | def test_redhat_storage_is_wrong_type(self, session): 28 | params = {'hosts': ['node1'], 'redhat_storage': "foo"} 29 | result = session.app.post_json("/api/mon/install/", params=params, 30 | expect_errors=True) 31 | message = result.json['message'] 32 | assert message.endswith('not of type boolean') 33 | assert 'redhat_storage' in message 34 | 35 | 36 | class TestErrors(object): 37 | 38 | def test_unavailable(self, session): 39 | result = session.app.get("/errors/unavailable/", 40 | expect_errors=True) 41 | message = result.json['message'] 42 | assert message == 'service unavailable' 43 | 44 | def test_forbidden(self, session): 45 | result = session.app.get("/errors/forbidden/", 46 | expect_errors=True) 47 | message = result.json['message'] 48 | assert message == 'forbidden' 49 | 50 | def test_invalid(self, session): 51 | result = session.app.get("/errors/forbidden/", 52 | expect_errors=True) 53 | message = result.json['message'] 54 | assert message == 'forbidden' 55 | -------------------------------------------------------------------------------- /tests/functional/nightly-centos7/vagrant_variables.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # DEPLOY CONTAINERIZED DAEMONS 4 | docker: false 5 | 6 | # DEFINE THE NUMBER OF VMS TO RUN 7 | mon_vms: 2 8 | osd_vms: 1 9 | mds_vms: 0 10 | rgw_vms: 1 11 | nfs_vms: 0 12 | rbd_mirror_vms: 0 13 | client_vms: 1 14 | iscsi_gw_vms: 0 15 | 16 | # SUBNETS TO USE FOR THE VMS 17 | public_subnet: 192.168.3 18 | cluster_subnet: 192.168.4 19 | 20 | # MEMORY 21 | # set 1024 for CentOS 22 | memory: 512 23 | 24 | # Ethernet interface name 25 | # use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial 26 | eth: 'eth1' 27 | 28 | # VAGRANT BOX 29 | # Ceph boxes are *strongly* suggested. They are under better control and will 30 | # not get updated frequently unless required for build systems. These are (for 31 | # now): 32 | # 33 | # * ceph/ubuntu-xenial 34 | # 35 | # Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 36 | # CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet 37 | # libvirt CentOS: centos/7 38 | # parallels Ubuntu: parallels/ubuntu-14.04 39 | # Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' 40 | # For more boxes have a look at: 41 | # - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= 42 | # - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ 43 | vagrant_box: centos/7 44 | client_vagrant_box: centos/7 45 | #ssh_private_key_path: "~/.ssh/id_rsa" 46 | # The sync directory changes based on vagrant box 47 | # Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant 48 | #vagrant_sync_dir: /home/vagrant/sync 49 | #vagrant_sync_dir: / 50 | # Disables synced folder creation. Not needed for testing, will skip mounting 51 | # the vagrant directory on the remote box regardless of the provider. 52 | vagrant_disable_synced_folder: true 53 | # VAGRANT URL 54 | # This is a URL to download an image from an alternate location. vagrant_box 55 | # above should be set to the filename of the image. 56 | # Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box 57 | # Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box 58 | # vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box 59 | -------------------------------------------------------------------------------- /tests/functional/nightly-xenial/vagrant_variables.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # DEPLOY CONTAINERIZED DAEMONS 4 | docker: false 5 | 6 | # DEFINE THE NUMBER OF VMS TO RUN 7 | mon_vms: 2 8 | osd_vms: 1 9 | mds_vms: 0 10 | rgw_vms: 1 11 | nfs_vms: 0 12 | rbd_mirror_vms: 0 13 | client_vms: 1 14 | iscsi_gw_vms: 0 15 | 16 | # SUBNETS TO USE FOR THE VMS 17 | public_subnet: 192.168.3 18 | cluster_subnet: 192.168.4 19 | 20 | # MEMORY 21 | # set 1024 for CentOS 22 | memory: 512 23 | 24 | # Ethernet interface name 25 | # use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial 26 | eth: 'eth1' 27 | 28 | # VAGRANT BOX 29 | # Ceph boxes are *strongly* suggested. They are under better control and will 30 | # not get updated frequently unless required for build systems. These are (for 31 | # now): 32 | # 33 | # * ceph/ubuntu-xenial 34 | # 35 | # Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 36 | # CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet 37 | # libvirt CentOS: centos/7 38 | # parallels Ubuntu: parallels/ubuntu-14.04 39 | # Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' 40 | # For more boxes have a look at: 41 | # - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= 42 | # - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ 43 | vagrant_box: ceph/ubuntu-xenial 44 | client_vagrant_box: centos/7 45 | #ssh_private_key_path: "~/.ssh/id_rsa" 46 | # The sync directory changes based on vagrant box 47 | # Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant 48 | #vagrant_sync_dir: /home/vagrant/sync 49 | #vagrant_sync_dir: / 50 | # Disables synced folder creation. Not needed for testing, will skip mounting 51 | # the vagrant directory on the remote box regardless of the provider. 52 | vagrant_disable_synced_folder: true 53 | # VAGRANT URL 54 | # This is a URL to download an image from an alternate location. vagrant_box 55 | # above should be set to the filename of the image. 56 | # Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box 57 | # Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box 58 | # vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box 59 | -------------------------------------------------------------------------------- /rpm/ktdreyer-ceph-installer.cfg: -------------------------------------------------------------------------------- 1 | config_opts['root'] = 'epel-7-x86_64' 2 | config_opts['target_arch'] = 'x86_64' 3 | config_opts['legal_host_arches'] = ('x86_64',) 4 | config_opts['chroot_setup_cmd'] = 'install @buildsys-build' 5 | config_opts['dist'] = 'el7' # only useful for --resultdir variable subst 6 | config_opts['releasever'] = '7' 7 | # See http://bugs.centos.org/view.php?id=7416 8 | config_opts['macros']['%dist'] = '.el7' 9 | 10 | config_opts['yum.conf'] = """ 11 | [main] 12 | keepcache=1 13 | debuglevel=2 14 | reposdir=/dev/null 15 | logfile=/var/log/yum.log 16 | retries=20 17 | obsoletes=1 18 | gpgcheck=0 19 | assumeyes=1 20 | syslog_ident=mock 21 | syslog_device= 22 | mdpolicy=group:primary 23 | 24 | # repos 25 | [base] 26 | name=BaseOS 27 | mirrorlist=http://mirrorlist.centos.org/?release=7&arch=x86_64&repo=os 28 | failovermethod=priority 29 | gpgkey=file:///usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-7 30 | gpgcheck=1 31 | 32 | [updates] 33 | name=updates 34 | enabled=1 35 | mirrorlist=http://mirrorlist.centos.org/?release=7&arch=x86_64&repo=updates 36 | failovermethod=priority 37 | gpgkey=file:///usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-7 38 | gpgcheck=1 39 | 40 | [epel] 41 | name=epel 42 | mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-7&arch=x86_64 43 | failovermethod=priority 44 | gpgkey=file:///usr/share/distribution-gpg-keys/epel/RPM-GPG-KEY-EPEL-7 45 | gpgcheck=1 46 | 47 | [extras] 48 | name=extras 49 | mirrorlist=http://mirrorlist.centos.org/?release=7&arch=x86_64&repo=extras 50 | failovermethod=priority 51 | gpgkey=file:///usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-7 52 | gpgcheck=1 53 | 54 | [testing] 55 | name=epel-testing 56 | enabled=0 57 | mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=testing-epel7&arch=x86_64 58 | failovermethod=priority 59 | 60 | 61 | [local] 62 | name=local 63 | baseurl=http://kojipkgs.fedoraproject.org/repos/epel7-build/latest/x86_64/ 64 | cost=2000 65 | enabled=0 66 | 67 | [epel-debug] 68 | name=epel-debug 69 | mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-debug-7&arch=x86_64 70 | failovermethod=priority 71 | enabled=0 72 | 73 | [ktdreyer-ceph-installer] 74 | name=Copr repo for ceph-installer owned by ktdreyer 75 | baseurl=https://copr-be.cloud.fedoraproject.org/results/ktdreyer/ceph-installer/epel-7-$basearch/ 76 | type=rpm-md 77 | skip_if_unavailable=True 78 | gpgcheck=1 79 | gpgkey=https://copr-be.cloud.fedoraproject.org/results/ktdreyer/ceph-installer/pubkey.gpg 80 | repo_gpgcheck=0 81 | enabled=1 82 | enabled_metadata=1 83 | """ 84 | -------------------------------------------------------------------------------- /ceph_installer/tests/test_hooks.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from ceph_installer import hooks 3 | from sqlalchemy.exc import OperationalError 4 | 5 | 6 | class FakeState(object): 7 | 8 | def __init__(self, **kw): 9 | for k, v in kw.items(): 10 | setattr(self, k, v) 11 | 12 | 13 | class TestAnsibleExists(object): 14 | 15 | def test_ansible_is_there(self, monkeypatch): 16 | monkeypatch.setattr( 17 | hooks, 'which', 18 | lambda x: '/usr/bin/ansible-playbook') 19 | assert hooks.ansible_exists() is None 20 | 21 | def test_ansible_is_not_there(self, monkeypatch): 22 | monkeypatch.setattr( 23 | hooks, 'which', 24 | lambda x: None) 25 | with pytest.raises(hooks.SystemCheckError): 26 | hooks.ansible_exists() 27 | 28 | 29 | class TestCeleryHasWorkers(object): 30 | 31 | def test_celery_has_workers(self, monkeypatch): 32 | stats = lambda: {'value': 'some stat'} 33 | monkeypatch.setattr( 34 | hooks, 'inspect', 35 | lambda: FakeState(stats=stats)) 36 | assert hooks.celery_has_workers() is None 37 | 38 | def test_celery_has_no_workers(self, monkeypatch): 39 | monkeypatch.setattr( 40 | hooks, 'inspect', 41 | lambda: FakeState(stats=lambda: None)) 42 | with pytest.raises(hooks.SystemCheckError): 43 | hooks.celery_has_workers() 44 | 45 | 46 | class TestRabbitMQIsRunning(object): 47 | 48 | def test_is_running(self, monkeypatch): 49 | def errors(): raise IOError 50 | monkeypatch.setattr( 51 | hooks, 'celery_has_workers', 52 | errors) 53 | with pytest.raises(hooks.SystemCheckError): 54 | assert hooks.rabbitmq_is_running() 55 | 56 | def test_is_not_running(self, monkeypatch): 57 | monkeypatch.setattr( 58 | hooks, 'celery_has_workers', 59 | lambda: None) 60 | assert hooks.rabbitmq_is_running() is None 61 | 62 | 63 | class TestDBConnection(object): 64 | 65 | def test_is_connected(self, monkeypatch): 66 | monkeypatch.setattr( 67 | hooks.models, 'Task', 68 | FakeState(get=lambda x: None)) 69 | assert hooks.database_connection() is None 70 | 71 | def test_is_not_running(self, monkeypatch): 72 | def errors(x): raise OperationalError(None, None, None, None) 73 | monkeypatch.setattr( 74 | hooks.models, 'Task', 75 | FakeState(get=errors)) 76 | with pytest.raises(hooks.SystemCheckError): 77 | hooks.database_connection() is None 78 | -------------------------------------------------------------------------------- /ceph_installer/process.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import tempfile 4 | import logging 5 | import json 6 | 7 | from ceph_installer.util import which, get_ceph_ansible_path 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | def make_ansible_command(hosts_file, identifier, extra_vars=None, tags='', skip_tags='', playbook='site.yml.sample', **kw): 13 | """ 14 | This utility will compose the command needed to run ansible, capture its 15 | stdout and stderr to a file 16 | ``verbose``: Optional keyword argument, to flag the need for increased 17 | verbosity when running ansible 18 | """ 19 | if kw.get('verbose'): 20 | verbose_flag = '-vvvv' 21 | else: 22 | verbose_flag = '-v' 23 | ceph_ansible_path = get_ceph_ansible_path() 24 | playbook = os.path.join(ceph_ansible_path, playbook) 25 | ansible_path = which('ansible-playbook') 26 | if not extra_vars: 27 | extra_vars = dict() 28 | 29 | extra_vars = json.dumps(extra_vars) 30 | 31 | cmd = [ 32 | ansible_path, verbose_flag, '-u', 'ceph-installer', 33 | playbook, '-i', hosts_file, "--extra-vars", extra_vars, 34 | ] 35 | 36 | if tags: 37 | cmd.extend(['--tags', tags]) 38 | 39 | if skip_tags: 40 | cmd.extend(['--skip-tags', skip_tags]) 41 | 42 | return cmd 43 | 44 | 45 | def temp_file(identifier, std): 46 | return tempfile.NamedTemporaryFile( 47 | prefix="{identifier}_{std}".format( 48 | identifier=identifier, std=std), delete=False 49 | ) 50 | 51 | 52 | def run(arguments, send_input=None, **kwargs): 53 | """ 54 | A small helper to run a system command using ``subprocess.Popen``. 55 | 56 | This returns the output of the command and the return code of the process 57 | in a tuple:: 58 | 59 | (stdout, stderr, returncode) 60 | """ 61 | env = os.environ.copy() 62 | env["ANSIBLE_HOST_KEY_CHECKING"] = "False" 63 | # preserves newlines in ansible output 64 | env["ANSIBLE_STDOUT_CALLBACK"] = "debug" 65 | logger.info('Running command: %s' % ' '.join(arguments)) 66 | process = subprocess.Popen( 67 | arguments, 68 | stdout=subprocess.PIPE, 69 | stderr=subprocess.PIPE, 70 | stdin=subprocess.PIPE, 71 | env=env, 72 | **kwargs) 73 | if send_input: 74 | out, err = process.communicate(send_input) 75 | else: 76 | out, err = process.communicate() 77 | 78 | # Ensure we are dealing with strings (might get a None) and decode them 79 | out = str(out).decode('utf-8', 'replace') 80 | err = str(err).decode('utf-8', 'replace') 81 | return out, err, process.returncode 82 | -------------------------------------------------------------------------------- /ceph_installer/tests/test_tasks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from ceph_installer import models, tasks 3 | 4 | 5 | class TestCallAnsible(object): 6 | 7 | def setup(self): 8 | models.Task( 9 | identifier='aaaa', 10 | endpoint='/api/test/', 11 | ) 12 | models.commit() 13 | 14 | def test_set_exit_code_on_error(self, session, monkeypatch): 15 | monkeypatch.setattr( 16 | tasks.process, 17 | 'make_ansible_command', 18 | lambda *a, **kw: ['echo'] 19 | ) 20 | tasks.call_ansible.apply(args=([], 'aaaa')).get() 21 | task = models.Task.get(1) 22 | assert task.exit_code == -1 23 | 24 | def test_set_exception_on_stderr(self, session, monkeypatch): 25 | def error(*a, **kw): raise OSError('a severe error') 26 | monkeypatch.setattr(tasks.process, 'run', error) 27 | monkeypatch.setattr( 28 | tasks.process, 29 | 'make_ansible_command', 30 | lambda *a, **kw: ['echo'] 31 | ) 32 | tasks.call_ansible.apply(args=([], 'aaaa')).get() 33 | task = models.Task.get(1) 34 | assert task.stderr == 'a severe error' 35 | 36 | def test_handle_unicode_on_stderr(self, session, monkeypatch): 37 | monkeypatch.setattr( 38 | tasks.process, 'run', 39 | lambda *a, **kw: ['stdout', 'ᓆ'.decode('utf-8', 'replace'), 1]) 40 | monkeypatch.setattr( 41 | tasks.process, 42 | 'make_ansible_command', 43 | lambda *a, **kw: ['echo'] 44 | ) 45 | tasks.call_ansible.apply(args=([], 'aaaa')).get() 46 | task = models.Task.get(1) 47 | assert task.stderr == u'\u14c6' 48 | 49 | def test_handle_unicode_on_stdout(self, session, monkeypatch): 50 | monkeypatch.setattr( 51 | tasks.process, 'run', 52 | lambda *a, **kw: ['£'.decode('utf-8', 'replace'), 'stderr', 1]) 53 | monkeypatch.setattr( 54 | tasks.process, 55 | 'make_ansible_command', 56 | lambda *a, **kw: ['echo'] 57 | ) 58 | tasks.call_ansible.apply(args=([], 'aaaa')).get() 59 | task = models.Task.get(1) 60 | assert task.stdout == u'\xa3' 61 | 62 | def test_process_a_request_that_is_none(self, session, monkeypatch): 63 | monkeypatch.setattr( 64 | tasks.process, 65 | 'make_ansible_command', 66 | lambda *a, **kw: ['echo'] 67 | ) 68 | tasks.call_ansible.apply(args=([], 'aaaa')).get() 69 | task = models.Task.get(1) 70 | assert task.http_method == '' 71 | assert task.user_agent == '' 72 | assert task.request == '' 73 | -------------------------------------------------------------------------------- /tests/functional/playbooks/roles/installer/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: install epel 4 | sudo: yes 5 | yum: 6 | name: epel-release 7 | state: present 8 | update_cache: yes 9 | 10 | - name: Install RPM requirements 11 | sudo: yes 12 | yum: 13 | name: "{{ item }}" 14 | state: present 15 | with_items: 16 | - git 17 | - python-devel 18 | - python-pip 19 | - python-virtualenv 20 | - ansible 21 | - redhat-lsb-core 22 | - gcc 23 | - gcc-c++ 24 | - openssl-devel 25 | - libffi-devel 26 | when: ansible_pkg_mgr == "yum" 27 | 28 | # we must use curl instead of ansible's uri module because SNI support in 29 | # Python is only available in 2.7.9 and later, and most supported distributions 30 | # don't have that version, so a request to https fails. 31 | - name: fetch ceph-installer development repo file 32 | command: 'curl -L https://shaman.ceph.com/api/repos/ceph-installer/{{ installer_dev_branch }}/{{ installer_dev_commit }}/{{ ansible_distribution | lower }}/{{ ansible_distribution_major_version }}/repo?arch=noarch' 33 | register: installer_yum_repo 34 | 35 | - name: add ceph-installer development repository 36 | sudo: yes 37 | copy: 38 | content: "{{ installer_yum_repo.stdout }}" 39 | dest: /etc/yum.repos.d/ceph-installer.repo 40 | owner: root 41 | group: root 42 | mode: 0644 43 | 44 | # we must use curl instead of ansible's uri module because SNI support in 45 | # Python is only available in 2.7.9 and later, and most supported distributions 46 | # don't have that version, so a request to https fails. 47 | - name: fetch ceph-ansible development repo file 48 | command: 'curl -L https://shaman.ceph.com/api/repos/ceph-ansible/{{ ceph_ansible_dev_branch }}/{{ ceph_ansible_dev_commit }}/{{ ansible_distribution | lower }}/{{ ansible_distribution_major_version }}/repo?arch=noarch' 49 | register: ceph_ansible_yum_repo 50 | 51 | - name: add ceph-installer development repository 52 | sudo: yes 53 | copy: 54 | content: "{{ ceph_ansible_yum_repo.stdout }}" 55 | dest: /etc/yum.repos.d/ceph-ansible.repo 56 | owner: root 57 | group: root 58 | mode: 0644 59 | 60 | - name: install a dev packages repo 61 | sudo: yes 62 | template: 63 | src: "templates/dev_repos.j2" 64 | dest: "/etc/yum.repos.d/dev.repo" 65 | owner: root 66 | group: root 67 | mode: 0644 68 | 69 | - name: purge yum cache 70 | sudo: yes 71 | command: yum clean all 72 | 73 | - name: install ceph-installer 74 | sudo: yes 75 | yum: 76 | name: ceph-installer 77 | state: present 78 | disable_gpg_check: true 79 | 80 | - name: ensure ceph-installer is running 81 | sudo: yes 82 | service: 83 | name: ceph-installer 84 | state: running 85 | enabled: true 86 | -------------------------------------------------------------------------------- /ceph_installer/tasks.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from datetime import datetime 4 | from celery import shared_task 5 | from ceph_installer import util, models, process 6 | 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | @shared_task 12 | def call_ansible(inventory, identifier, tags="", skip_tags="", extra_vars=None, playbook="site.yml.sample", **kw): 13 | """ 14 | This task builds an ansible-playbook command and runs it. 15 | 16 | ``inventory``: A list of tuples that details an ansible inventory. For example: 17 | [('mons', ['mon1.host', 'mon2.host']), ('osds', ['osd1.host'])] 18 | ``tags``: The tags as a comma-delimeted string that represents all the tags 19 | this ansible call should follow. For example "package-install, other-tag" 20 | ``skip_tags``: The tags as a comma-delimeted string that represents all the tags 21 | this ansible call should skip. For example "package-install, other-tag" 22 | ``identifier``: The UUID identifer for the task object so this function can capture process 23 | metadata and persist it to the database 24 | ``verbose``: Optional keyword argument, to flag the need for increased verbosity 25 | when running ansible 26 | """ 27 | verbose = kw.get('verbose', False) 28 | if not extra_vars: 29 | extra_vars = dict() 30 | hosts_file = util.generate_inventory_file(inventory, identifier) 31 | command = process.make_ansible_command( 32 | hosts_file, identifier, tags=tags, extra_vars=extra_vars, 33 | skip_tags=skip_tags, playbook=playbook, verbose=verbose 34 | ) 35 | task = models.Task.query.filter_by(identifier=identifier).first() 36 | task.command = ' '.join(command) 37 | task.started = datetime.now() 38 | # force a commit here so we can reference this command later if it fails 39 | models.commit() 40 | working_dir = util.get_ceph_ansible_path() 41 | # ansible depends on relative pathing to figure out how to load 42 | # plugins, among other things. Setting the current working directory 43 | # for this subprocess call to the directory where the playbook resides 44 | # allows ansible to properly find action plugins defined in ceph-ansible. 45 | kwargs = dict(cwd=working_dir) 46 | try: 47 | out, err, exit_code = process.run(command, **kwargs) 48 | except Exception as error: 49 | task.succeeded = False 50 | task.exit_code = -1 51 | task.stderr = str(error) 52 | logger.exception('failed to run command') 53 | else: 54 | task.succeeded = not exit_code 55 | task.exit_code = exit_code 56 | task.stdout = out 57 | task.stderr = err 58 | 59 | task.ended = datetime.now() 60 | models.commit() 61 | -------------------------------------------------------------------------------- /ceph_installer/controllers/osd.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pecan import expose, request 4 | from pecan.ext.notario import validate 5 | from uuid import uuid4 6 | 7 | from ceph_installer.controllers import error 8 | from ceph_installer.tasks import call_ansible 9 | from ceph_installer import schemas 10 | from ceph_installer import models 11 | from ceph_installer import util 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class OSDController(object): 17 | 18 | @expose('json') 19 | def index(self): 20 | # TODO: allow some autodiscovery here so that clients can see what is 21 | # available 22 | return dict() 23 | 24 | @expose(generic=True, template='json') 25 | def install(self): 26 | error(405) 27 | 28 | @install.when(method='POST', template='json') 29 | @validate(schemas.install_schema, handler="/errors/schema") 30 | def install_post(self): 31 | hosts = request.json.get('hosts') 32 | verbose_ansible = request.json.get('verbose', False) 33 | extra_vars = util.get_install_extra_vars(request.json) 34 | identifier = str(uuid4()) 35 | task = models.Task( 36 | request=request, 37 | identifier=identifier, 38 | endpoint=request.path, 39 | ) 40 | # we need an explicit commit here because the command may finish before 41 | # we conclude this request 42 | models.commit() 43 | kwargs = dict( 44 | extra_vars=extra_vars, 45 | tags="package-install", 46 | verbose=verbose_ansible, 47 | ) 48 | call_ansible.apply_async( 49 | args=([('osds', hosts)], identifier), 50 | kwargs=kwargs 51 | ) 52 | 53 | return task 54 | 55 | @expose(generic=True, template='json') 56 | def configure(self): 57 | error(405) 58 | 59 | @configure.when(method='POST', template='json') 60 | @validate(schemas.osd_configure_schema, handler="/errors/schema") 61 | def configure_post(self): 62 | hosts = [request.json['host']] 63 | monitor_hosts = util.parse_monitors(request.json["monitors"]) 64 | verbose_ansible = request.json.get('verbose', False) 65 | # even with configuring we need to tell ceph-ansible 66 | # if we're working with upstream ceph or red hat ceph storage 67 | extra_vars = util.get_osd_configure_extra_vars(request.json) 68 | if 'verbose' in extra_vars: 69 | del extra_vars['verbose'] 70 | if 'conf' in extra_vars: 71 | extra_vars['ceph_conf_overrides'] = request.json['conf'] 72 | del extra_vars['conf'] 73 | identifier = str(uuid4()) 74 | task = models.Task( 75 | request=request, 76 | identifier=identifier, 77 | endpoint=request.path, 78 | ) 79 | # we need an explicit commit here because the command may finish before 80 | # we conclude this request 81 | models.commit() 82 | kwargs = dict( 83 | extra_vars=extra_vars, 84 | skip_tags="package-install", 85 | playbook="infrastructure-playbooks/osd-configure.yml", 86 | verbose=verbose_ansible, 87 | ) 88 | call_ansible.apply_async( 89 | args=([('osds', hosts), ('mons', monitor_hosts)], identifier), 90 | kwargs=kwargs 91 | ) 92 | 93 | return task 94 | -------------------------------------------------------------------------------- /ceph_installer/models/__init__.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from sqlalchemy import create_engine, MetaData, event 3 | from sqlalchemy.orm import scoped_session, sessionmaker, object_session, mapper 4 | from sqlalchemy.ext.declarative import declarative_base 5 | from pecan import conf 6 | 7 | 8 | class _EntityBase(object): 9 | """ 10 | A custom declarative base that provides some Elixir-inspired shortcuts. 11 | """ 12 | 13 | @classmethod 14 | def filter_by(cls, *args, **kwargs): 15 | return cls.query.filter_by(*args, **kwargs) 16 | 17 | @classmethod 18 | def get(cls, *args, **kwargs): 19 | return cls.query.get(*args, **kwargs) 20 | 21 | def flush(self, *args, **kwargs): 22 | object_session(self).flush([self], *args, **kwargs) 23 | 24 | def delete(self, *args, **kwargs): 25 | object_session(self).delete(self, *args, **kwargs) 26 | 27 | def as_dict(self): 28 | return dict((k, v) for k, v in self.__dict__.items() 29 | if not k.startswith('_')) 30 | 31 | def update_from_json(self, data): 32 | """ 33 | We received a JSON blob with updated metadata information 34 | that needs to update some fields 35 | """ 36 | for key in data.keys(): 37 | setattr(self, key, data[key]) 38 | 39 | 40 | Session = scoped_session(sessionmaker()) 41 | metadata = MetaData() 42 | Base = declarative_base(cls=_EntityBase) 43 | Base.query = Session.query_property() 44 | 45 | 46 | # Listeners: 47 | 48 | @event.listens_for(mapper, 'init') 49 | def auto_add(target, args, kwargs): 50 | Session.add(target) 51 | 52 | 53 | def update_timestamp(mapper, connection, target): 54 | """ 55 | Automate the 'modified' attribute when a model changes 56 | """ 57 | target.modified = datetime.datetime.utcnow() 58 | 59 | 60 | # Utilities: 61 | 62 | def get_or_create(model, **kwargs): 63 | instance = model.filter_by(**kwargs).first() 64 | if instance: 65 | return instance 66 | else: 67 | instance = model(**kwargs) 68 | commit() 69 | return instance 70 | 71 | 72 | def init_model(): 73 | """ 74 | This is a stub method which is called at application startup time. 75 | 76 | If you need to bind to a parse database configuration, set up tables or 77 | ORM classes, or perform any database initialization, this is the 78 | recommended place to do it. 79 | 80 | For more information working with databases, and some common recipes, 81 | see http://pecan.readthedocs.org/en/latest/databases.html 82 | 83 | For creating all metadata you would use:: 84 | 85 | Base.metadata.create_all(conf.sqlalchemy.engine) 86 | 87 | """ 88 | conf.sqlalchemy.engine = _engine_from_config(conf.sqlalchemy) 89 | Session.configure(bind=conf.sqlalchemy.engine) 90 | 91 | 92 | def _engine_from_config(configuration): 93 | configuration = dict(configuration) 94 | url = configuration.pop('url') 95 | return create_engine(url, **configuration) 96 | 97 | 98 | def start(): 99 | Session() 100 | metadata.bind = conf.sqlalchemy.engine 101 | 102 | 103 | def start_read_only(): 104 | start() 105 | 106 | 107 | def commit(): 108 | Session.commit() 109 | 110 | 111 | def rollback(): 112 | Session.rollback() 113 | 114 | 115 | def clear(): 116 | Session.remove() 117 | Session.close() 118 | 119 | 120 | def flush(): 121 | Session.flush() 122 | 123 | 124 | from tasks import Task # noqa 125 | -------------------------------------------------------------------------------- /ceph_installer/tests/controllers/test_agent.py: -------------------------------------------------------------------------------- 1 | from ceph_installer.controllers import agent 2 | 3 | 4 | class TestAgentController(object): 5 | 6 | def test_index_get_is_not_allowed(self, session): 7 | result = session.app.get("/api/agent/", expect_errors=True) 8 | assert result.status_int == 405 9 | 10 | def test_index_incorrect_schema(self, session): 11 | data = dict(hosts=["google.com"], bogus="foo") 12 | result = session.app.post_json("/api/agent/", params=data, 13 | expect_errors=True) 14 | assert result.status_int == 400 15 | 16 | def test_index_post_works_with_right_schema(self, session, monkeypatch): 17 | monkeypatch.setattr(agent.call_ansible, 'apply_async', lambda args, kwargs: None) 18 | data = dict(hosts=["node1"]) 19 | result = session.app.post_json("/api/agent/", params=data) 20 | assert result.json['endpoint'] == '/api/agent/' 21 | assert result.json['identifier'] is not None 22 | 23 | def test_index_post_is_not_denied_remotely(self, session, monkeypatch): 24 | monkeypatch.setattr(agent.call_ansible, 'apply_async', lambda args, kwargs: None) 25 | data = dict(hosts=["node1"]) 26 | result = session.app.post_json( 27 | "/api/agent/", 28 | params=data, 29 | expect_errors=True, 30 | extra_environ=dict(REMOTE_ADDR='192.168.1.1') 31 | ) 32 | assert result.status_int == 200 33 | assert result.json['endpoint'] == '/api/agent/' 34 | assert result.json['identifier'] is not None 35 | 36 | def test_invalid_master_value_is_detected(self, session, monkeypatch): 37 | data = dict(hosts=["google.com"], master=1) 38 | result = session.app.post_json("/api/agent/", params=data, 39 | expect_errors=True) 40 | assert result.status_int == 400 41 | assert result.json['message'].endswith('not of type string') 42 | 43 | def test_master_is_defined_and_used(self, session, monkeypatch, argtest): 44 | monkeypatch.setattr(agent.call_ansible, 'apply_async', argtest) 45 | data = dict(hosts=["node1"], master='installer.host') 46 | session.app.post_json("/api/agent/", params=data) 47 | # argtest.kwargs['kwargs'] may look a bit redundant, but celery tasks 48 | # require that same signature as the 'recorder' fixture we have to 49 | # capture args and kwargs 50 | ansible_extra_args = argtest.kwargs['kwargs']['extra_vars'] 51 | assert ansible_extra_args['agent_master_host'] == 'installer.host' 52 | 53 | def test_master_is_not_defined_and_is_inferred(self, session, monkeypatch, argtest): 54 | monkeypatch.setattr(agent.call_ansible, 'apply_async', argtest) 55 | data = dict(hosts=["node1"]) 56 | session.app.post_json("/api/agent/", params=data) 57 | ansible_extra_args = argtest.kwargs['kwargs']['extra_vars'] 58 | assert ansible_extra_args['agent_master_host'] == 'localhost' 59 | 60 | 61 | class TestAgentVerbose(object): 62 | 63 | def test_install_verbose(self, session, monkeypatch, argtest): 64 | monkeypatch.setattr( 65 | agent.call_ansible, 'apply_async', argtest) 66 | data = {"hosts": ["node1"], "verbose": True} 67 | session.app.post_json("/api/agent/", params=data) 68 | kwargs = argtest.kwargs['kwargs'] 69 | assert kwargs['verbose'] is True 70 | 71 | def test_install_non_verbose(self, session, monkeypatch, argtest): 72 | monkeypatch.setattr( 73 | agent.call_ansible, 'apply_async', argtest) 74 | data = {"hosts": ["node1"]} 75 | session.app.post_json("/api/agent/", params=data) 76 | kwargs = argtest.kwargs['kwargs'] 77 | assert kwargs['verbose'] is False 78 | -------------------------------------------------------------------------------- /ceph_installer/cli/decorators.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | import traceback 4 | from functools import wraps 5 | 6 | 7 | def catches(catch=None, handler=None, exit=True, handle_all=False, logger=None): 8 | """ 9 | Very simple decorator that tries any of the exception(s) passed in as 10 | a single exception class or tuple (containing multiple ones) returning the 11 | exception message and optionally handling the problem if it raises with the 12 | handler if it is provided. 13 | 14 | So instead of doing something like this:: 15 | 16 | def bar(): 17 | try: 18 | some_call() 19 | print "Success!" 20 | except TypeError, exc: 21 | print "Error while handling some call: %s" % exc 22 | sys.exit(1) 23 | 24 | You would need to decorate it like this to have the same effect:: 25 | 26 | @catches(TypeError) 27 | def bar(): 28 | some_call() 29 | print "Success!" 30 | 31 | If multiple exceptions need to be caught they need to be provided as a 32 | tuple:: 33 | 34 | @catches((TypeError, AttributeError)) 35 | def bar(): 36 | some_call() 37 | print "Success!" 38 | 39 | If adding a handler, it should accept a single argument, which would be the 40 | exception that was raised, it would look like:: 41 | 42 | def my_handler(exc): 43 | print 'Handling exception %s' % str(exc) 44 | raise SystemExit 45 | 46 | @catches(KeyboardInterrupt, handler=my_handler) 47 | def bar(): 48 | some_call() 49 | 50 | Note that the handler needs to raise its SystemExit if it wants to halt 51 | execution, otherwise the decorator would continue as a normal try/except 52 | block. 53 | 54 | 55 | :param catch: A tuple with one (or more) Exceptions to catch 56 | :param handler: Optional handler to have custom handling of exceptions 57 | :param exit: Raise a ``SystemExit`` after handling exceptions 58 | :param handle_all: Handle all other exceptions via logging. 59 | """ 60 | catch = catch or Exception 61 | logger = logger or logging.getLogger('ceph-installer') 62 | 63 | def decorate(f): 64 | 65 | @wraps(f) 66 | def newfunc(*a, **kw): 67 | exit_from_catch = False 68 | try: 69 | return f(*a, **kw) 70 | except catch as e: 71 | if handler: 72 | return handler(e) 73 | else: 74 | logger.error(make_exception_message(e)) 75 | 76 | if exit: 77 | exit_from_catch = True 78 | sys.exit(1) 79 | except Exception: # anything else, no need to save the exception as a variable 80 | if handle_all is False: # re-raise if we are not supposed to handle everything 81 | raise 82 | # Make sure we don't spit double tracebacks if we are raising 83 | # SystemExit from the `except catch` block 84 | 85 | if exit_from_catch: 86 | sys.exit(1) 87 | 88 | str_failure = traceback.format_exc() 89 | for line in str_failure.split('\n'): 90 | logger.error("%s" % line) 91 | sys.exit(1) 92 | 93 | return newfunc 94 | 95 | return decorate 96 | 97 | # 98 | # Decorator helpers 99 | # 100 | 101 | 102 | def make_exception_message(exc): 103 | """ 104 | An exception is passed in and this function 105 | returns the proper string depending on the result 106 | so it is readable enough. 107 | """ 108 | if str(exc): 109 | return '%s: %s\n' % (exc.__class__.__name__, exc) 110 | else: 111 | return '%s\n' % (exc.__class__.__name__) 112 | -------------------------------------------------------------------------------- /ceph_installer/controllers/rgw.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pecan import expose, request 4 | from pecan.ext.notario import validate 5 | from uuid import uuid4 6 | 7 | from ceph_installer.controllers import error 8 | from ceph_installer.tasks import call_ansible 9 | from ceph_installer import schemas 10 | from ceph_installer import models 11 | from ceph_installer import util 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class RGWController(object): 17 | 18 | @expose('json') 19 | def index(self): 20 | # TODO: allow some autodiscovery here so that clients can see what is 21 | # available 22 | return dict() 23 | 24 | @expose(generic=True, template='json') 25 | def install(self): 26 | error(405) 27 | 28 | @install.when(method='POST', template='json') 29 | @validate(schemas.install_schema, handler="/errors/schema") 30 | def install_post(self): 31 | hosts = request.json.get('hosts') 32 | identifier = str(uuid4()) 33 | extra_vars = util.get_install_extra_vars(request.json) 34 | verbose_ansible = request.json.get('verbose', False) 35 | task = models.Task( 36 | request=request, 37 | identifier=identifier, 38 | endpoint=request.path, 39 | ) 40 | # we need an explicit commit here because the command may finish before 41 | # we conclude this request 42 | models.commit() 43 | kwargs = dict( 44 | extra_vars=extra_vars, 45 | tags="package-install", 46 | verbose=verbose_ansible, 47 | ) 48 | call_ansible.apply_async( 49 | args=([('rgws', hosts)], identifier), 50 | kwargs=kwargs, 51 | ) 52 | 53 | return task 54 | 55 | @expose(generic=True, template='json') 56 | def configure(self): 57 | error(405) 58 | 59 | @configure.when(method='POST', template='json') 60 | @validate(schemas.rgw_configure_schema, handler="/errors/schema") 61 | def configure_post(self): 62 | hosts = [request.json['host']] 63 | # even with configuring we need to tell ceph-ansible 64 | # if we're working with upstream ceph or red hat ceph storage 65 | verbose_ansible = request.json.get('verbose', False) 66 | extra_vars = util.get_install_extra_vars(request.json) 67 | monitor_hosts = util.parse_monitors(request.json["monitors"]) 68 | # this update will take everything in the ``request.json`` body and 69 | # just pass it in as extra-vars. That is the reason why optional values 70 | # like "calamari" are not looked up explicitly. If they are passed in 71 | # they will be used. 72 | extra_vars.update(request.json) 73 | if 'verbose' in extra_vars: 74 | del extra_vars['verbose'] 75 | if 'conf' in extra_vars: 76 | extra_vars['ceph_conf_overrides'] = request.json['conf'] 77 | del extra_vars['conf'] 78 | if "cluster_name" in extra_vars: 79 | extra_vars["cluster"] = extra_vars["cluster_name"] 80 | del extra_vars["cluster_name"] 81 | del extra_vars['host'] 82 | extra_vars.pop('interface', None) 83 | extra_vars.pop('address', None) 84 | identifier = str(uuid4()) 85 | task = models.Task( 86 | request=request, 87 | identifier=identifier, 88 | playbook="infrastructure-playbooks/rgw-standalone.yml", 89 | endpoint=request.path, 90 | ) 91 | # we need an explicit commit here because the command may finish before 92 | # we conclude this request 93 | models.commit() 94 | kwargs = dict( 95 | extra_vars=extra_vars, 96 | skip_tags="package-install", 97 | verbose=verbose_ansible, 98 | ) 99 | call_ansible.apply_async( 100 | args=([('rgws', hosts), ('mons', monitor_hosts)], identifier), 101 | kwargs=kwargs, 102 | ) 103 | 104 | return task 105 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import subprocess 3 | try: 4 | from setuptools import setup, find_packages, Command 5 | except ImportError: 6 | from ez_setup import use_setuptools 7 | use_setuptools() 8 | from setuptools import setup, find_packages, Command 9 | import re 10 | 11 | 12 | def read_module_contents(): 13 | with open('ceph_installer/__init__.py') as installer_init: 14 | return installer_init.read() 15 | 16 | module_file = read_module_contents() 17 | metadata = dict(re.findall("__([a-z]+)__\s*=\s*'([^']+)'", module_file)) 18 | long_description = open('README.rst').read() 19 | version = metadata['version'] 20 | 21 | 22 | class BumpCommand(Command): 23 | """ Bump the __version__ number and commit all changes. """ 24 | 25 | user_options = [('version=', 'v', 'version number to use')] 26 | 27 | def initialize_options(self): 28 | new_version = metadata['version'].split('.') 29 | new_version[-1] = str(int(new_version[-1]) + 1) # Bump the final part 30 | self.version = ".".join(new_version) 31 | 32 | def finalize_options(self): 33 | pass 34 | 35 | def run(self): 36 | 37 | try: 38 | print('old version: %s new version: %s' % 39 | (metadata['version'], self.version)) 40 | raw_input('Press enter to confirm, or ctrl-c to exit >') 41 | except KeyboardInterrupt: 42 | raise SystemExit("\nNot proceeding") 43 | 44 | old = "__version__ = '%s'" % metadata['version'] 45 | new = "__version__ = '%s'" % self.version 46 | 47 | module_file = read_module_contents() 48 | with open('ceph_installer/__init__.py', 'w') as fileh: 49 | fileh.write(module_file.replace(old, new)) 50 | 51 | # Commit everything with a standard commit message 52 | cmd = ['git', 'commit', '-a', '-m', 'version %s' % self.version] 53 | print(' '.join(cmd)) 54 | subprocess.check_call(cmd) 55 | 56 | 57 | class ReleaseCommand(Command): 58 | """ Tag and push a new release. """ 59 | 60 | user_options = [('sign', 's', 'GPG-sign the Git tag and release files')] 61 | 62 | def initialize_options(self): 63 | self.sign = False 64 | 65 | def finalize_options(self): 66 | pass 67 | 68 | def run(self): 69 | # Create Git tag 70 | tag_name = 'v%s' % version 71 | cmd = ['git', 'tag', '-a', tag_name, '-m', 'version %s' % version] 72 | if self.sign: 73 | cmd.append('-s') 74 | print(' '.join(cmd)) 75 | subprocess.check_call(cmd) 76 | 77 | # Push Git tag to origin remote 78 | cmd = ['git', 'push', 'origin', tag_name] 79 | print(' '.join(cmd)) 80 | subprocess.check_call(cmd) 81 | 82 | # Push package to pypi 83 | cmd = ['python', 'setup.py', 'sdist', 'upload'] 84 | if self.sign: 85 | cmd.append('--sign') 86 | print(' '.join(cmd)) 87 | subprocess.check_call(cmd) 88 | 89 | # Push master to the remote 90 | cmd = ['git', 'push', 'origin', 'master'] 91 | print(' '.join(cmd)) 92 | subprocess.check_call(cmd) 93 | 94 | 95 | setup( 96 | name='ceph-installer', 97 | version=version, 98 | description='An HTTP API that provides Ceph installation/configuration endpoints', 99 | long_description=long_description, 100 | author='', 101 | author_email='', 102 | scripts=['bin/ceph-installer', 'bin/ceph-installer-celery', 'bin/ceph-installer-gunicorn'], 103 | install_requires=[ 104 | 'celery<4.0.0', 105 | 'gunicorn', 106 | 'notario>=0.0.11', 107 | 'pecan>=1', 108 | 'pecan-notario', 109 | 'requests', 110 | 'sqlalchemy', 111 | 'tambo', 112 | ], 113 | zip_safe=False, 114 | include_package_data=True, 115 | packages=find_packages(exclude=['ez_setup']), 116 | entry_points=""" 117 | [pecan.command] 118 | populate=ceph_installer.commands.populate:PopulateCommand 119 | """, 120 | cmdclass={'bump': BumpCommand, 'release': ReleaseCommand}, 121 | ) 122 | -------------------------------------------------------------------------------- /selinux/ceph_installer.if: -------------------------------------------------------------------------------- 1 | 2 | ## policy for ceph_installer 3 | 4 | ######################################## 5 | ## 6 | ## Execute TEMPLATE in the ceph_installer domin. 7 | ## 8 | ## 9 | ## 10 | ## Domain allowed to transition. 11 | ## 12 | ## 13 | # 14 | interface(`ceph_installer_domtrans',` 15 | gen_require(` 16 | type ceph_installer_t, ceph_installer_exec_t; 17 | ') 18 | 19 | corecmd_search_bin($1) 20 | domtrans_pattern($1, ceph_installer_exec_t, ceph_installer_t) 21 | ') 22 | 23 | ######################################## 24 | ## 25 | ## Search ceph_installer lib directories. 26 | ## 27 | ## 28 | ## 29 | ## Domain allowed access. 30 | ## 31 | ## 32 | # 33 | interface(`ceph_installer_search_lib',` 34 | gen_require(` 35 | type ceph_installer_var_lib_t; 36 | ') 37 | 38 | allow $1 ceph_installer_var_lib_t:dir search_dir_perms; 39 | files_search_var_lib($1) 40 | ') 41 | 42 | ######################################## 43 | ## 44 | ## Read ceph_installer lib files. 45 | ## 46 | ## 47 | ## 48 | ## Domain allowed access. 49 | ## 50 | ## 51 | # 52 | interface(`ceph_installer_read_lib_files',` 53 | gen_require(` 54 | type ceph_installer_var_lib_t; 55 | ') 56 | 57 | files_search_var_lib($1) 58 | read_files_pattern($1, ceph_installer_var_lib_t, ceph_installer_var_lib_t) 59 | ') 60 | 61 | ######################################## 62 | ## 63 | ## Manage ceph_installer lib files. 64 | ## 65 | ## 66 | ## 67 | ## Domain allowed access. 68 | ## 69 | ## 70 | # 71 | interface(`ceph_installer_manage_lib_files',` 72 | gen_require(` 73 | type ceph_installer_var_lib_t; 74 | ') 75 | 76 | files_search_var_lib($1) 77 | manage_files_pattern($1, ceph_installer_var_lib_t, ceph_installer_var_lib_t) 78 | ') 79 | 80 | ######################################## 81 | ## 82 | ## Manage ceph_installer lib directories. 83 | ## 84 | ## 85 | ## 86 | ## Domain allowed access. 87 | ## 88 | ## 89 | # 90 | interface(`ceph_installer_manage_lib_dirs',` 91 | gen_require(` 92 | type ceph_installer_var_lib_t; 93 | ') 94 | 95 | files_search_var_lib($1) 96 | manage_dirs_pattern($1, ceph_installer_var_lib_t, ceph_installer_var_lib_t) 97 | ') 98 | 99 | ######################################## 100 | ## 101 | ## Execute ceph_installer server in the ceph_installer domain. 102 | ## 103 | ## 104 | ## 105 | ## Domain allowed to transition. 106 | ## 107 | ## 108 | # 109 | interface(`ceph_installer_systemctl',` 110 | gen_require(` 111 | type ceph_installer_t; 112 | type ceph_installer_unit_file_t; 113 | ') 114 | 115 | systemd_exec_systemctl($1) 116 | systemd_read_fifo_file_passwd_run($1) 117 | allow $1 ceph_installer_unit_file_t:file read_file_perms; 118 | allow $1 ceph_installer_unit_file_t:service manage_service_perms; 119 | 120 | ps_process_pattern($1, ceph_installer_t) 121 | ') 122 | 123 | 124 | ######################################## 125 | ## 126 | ## All of the rules required to administrate 127 | ## an ceph_installer environment 128 | ## 129 | ## 130 | ## 131 | ## Domain allowed access. 132 | ## 133 | ## 134 | ## 135 | ## 136 | ## Role allowed access. 137 | ## 138 | ## 139 | ## 140 | # 141 | interface(`ceph_installer_admin',` 142 | gen_require(` 143 | type ceph_installer_t; 144 | type ceph_installer_var_lib_t; 145 | type ceph_installer_unit_file_t; 146 | ') 147 | 148 | allow $1 ceph_installer_t:process { signal_perms }; 149 | ps_process_pattern($1, ceph_installer_t) 150 | 151 | tunable_policy(`deny_ptrace',`',` 152 | allow $1 ceph_installer_t:process ptrace; 153 | ') 154 | 155 | files_search_var_lib($1) 156 | admin_pattern($1, ceph_installer_var_lib_t) 157 | 158 | ceph_installer_systemctl($1) 159 | admin_pattern($1, ceph_installer_unit_file_t) 160 | allow $1 ceph_installer_unit_file_t:service all_service_perms; 161 | optional_policy(` 162 | systemd_passwd_agent_exec($1) 163 | systemd_read_fifo_file_passwd_run($1) 164 | ') 165 | ') 166 | -------------------------------------------------------------------------------- /ceph_installer/controllers/mon.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pecan import expose, request 4 | from pecan.ext.notario import validate 5 | from uuid import uuid4 6 | 7 | from ceph_installer.controllers import error 8 | from ceph_installer.tasks import call_ansible 9 | from ceph_installer import schemas 10 | from ceph_installer import models 11 | from ceph_installer import util 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | class MONController(object): 18 | 19 | @expose('json') 20 | def index(self): 21 | # TODO: allow some autodiscovery here so that clients can see what is 22 | # available 23 | return dict() 24 | 25 | @expose(generic=True, template='json') 26 | def install(self): 27 | error(405) 28 | 29 | @install.when(method='POST', template='json') 30 | @validate(schemas.mon_install_schema, handler="/errors/schema") 31 | def install_post(self): 32 | hosts = request.json.get('hosts') 33 | install_calamari = request.json.get('calamari', False) 34 | verbose_ansible = request.json.get('verbose', False) 35 | extra_vars = util.get_install_extra_vars(request.json) 36 | extra_vars['calamari'] = install_calamari 37 | identifier = str(uuid4()) 38 | task = models.Task( 39 | request=request, 40 | identifier=identifier, 41 | endpoint=request.path, 42 | ) 43 | # we need an explicit commit here because the command may finish before 44 | # we conclude this request 45 | models.commit() 46 | kwargs = dict( 47 | extra_vars=extra_vars, 48 | tags="package-install", 49 | verbose=verbose_ansible, 50 | ) 51 | 52 | call_ansible.apply_async( 53 | args=([('mons', hosts)], identifier), 54 | kwargs=kwargs, 55 | ) 56 | 57 | return task 58 | 59 | @expose(generic=True, template='json') 60 | def configure(self): 61 | error(405) 62 | 63 | @configure.when(method='POST', template='json') 64 | @validate(schemas.mon_configure_schema, handler="/errors/schema") 65 | def configure_post(self): 66 | monitor_mapping = dict(host=request.json['host']) 67 | 68 | # Only add interface and address if they exist 69 | for key in ['interface', 'address']: 70 | try: 71 | monitor_mapping[key] = request.json[key] 72 | except KeyError: 73 | pass 74 | hosts = util.parse_monitors([monitor_mapping]) 75 | verbose_ansible = request.json.get('verbose', False) 76 | monitors = request.json.get("monitors", []) 77 | monitors = util.validate_monitors(monitors, request.json["host"]) 78 | # even with configuring we need to tell ceph-ansible 79 | # if we're working with upstream ceph or red hat ceph storage 80 | extra_vars = util.get_install_extra_vars(request.json) 81 | # this update will take everything in the ``request.json`` body and 82 | # just pass it in as extra-vars. That is the reason why optional values 83 | # like "calamari" are not looked up explicitly. If they are passed in 84 | # they will be used. 85 | extra_vars.update(request.json) 86 | if 'verbose' in extra_vars: 87 | del extra_vars['verbose'] 88 | if 'conf' in extra_vars: 89 | extra_vars['ceph_conf_overrides'] = request.json['conf'] 90 | del extra_vars['conf'] 91 | if monitors: 92 | hosts.extend(util.parse_monitors(monitors)) 93 | del extra_vars['monitors'] 94 | if "cluster_name" in extra_vars: 95 | extra_vars["cluster"] = extra_vars["cluster_name"] 96 | del extra_vars["cluster_name"] 97 | del extra_vars['host'] 98 | extra_vars.pop('interface', None) 99 | extra_vars.pop('address', None) 100 | identifier = str(uuid4()) 101 | task = models.Task( 102 | request=request, 103 | identifier=identifier, 104 | endpoint=request.path, 105 | ) 106 | # we need an explicit commit here because the command may finish before 107 | # we conclude this request 108 | models.commit() 109 | kwargs = dict( 110 | extra_vars=extra_vars, 111 | skip_tags="package-install", 112 | verbose=verbose_ansible, 113 | ) 114 | call_ansible.apply_async( 115 | args=([('mons', hosts)], identifier), 116 | kwargs=kwargs, 117 | ) 118 | 119 | return task 120 | -------------------------------------------------------------------------------- /ceph_installer/cli/task.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | import sys 3 | import requests 4 | import time 5 | from textwrap import dedent 6 | from tambo import Transport 7 | 8 | from ceph_installer.cli import log, constants 9 | 10 | 11 | class Task(object): 12 | 13 | help = "/api/tasks/ operations" 14 | options = [] 15 | _help = dedent(""" 16 | Human-readable task information: stdout, stderr, and the ability to "poll" 17 | a task that waits until the command completes to be able to show the output 18 | in a readable way. 19 | 20 | Usage:: 21 | 22 | ceph-installer task $IDENTIFIER 23 | 24 | Options: 25 | 26 | --poll Poll until the task has completed (either on failure or success) 27 | stdout Retrieve the stdout output from the task 28 | stderr Retrieve the stderr output from the task 29 | command The actual command used to call ansible 30 | ended The timestamp (in UTC) when the command completed 31 | started The timestamp (in UTC) when the command started 32 | exit_code The shell exit status for the process 33 | succeeded Boolean value to indicate if process completed correctly 34 | """) 35 | 36 | def __init__(self, arguments): 37 | self.arguments = arguments 38 | self.tasks_url = path.join(constants.server_address, 'api/tasks') 39 | self.identifier = '' 40 | 41 | @property 42 | def request_url(self): 43 | url = path.join(self.tasks_url, self.identifier) 44 | # and add a trailing slash so that the request is done at the correct 45 | # canonical url 46 | if not url.endswith('/'): 47 | url = "%s/" % url 48 | return url 49 | 50 | def get(self, key): 51 | """ 52 | :arg key: any actual key that can be present in the JSON output 53 | """ 54 | log.info('requesting task at: %s' % self.request_url) 55 | response = requests.get(self.request_url) 56 | json = response.json() 57 | if response.status_code >= 400: 58 | return log.error(json['message']) 59 | try: 60 | value = json[key] 61 | log.info("%s: %s" % (key, value)) 62 | except KeyError: 63 | return log.warning('no value found for: "%s"' % key) 64 | 65 | def summary(self): 66 | response = requests.get(self.request_url) 67 | json = response.json() 68 | if response.status_code >= 400: 69 | log.error(json['message']) 70 | for k, v in json.items(): 71 | log.debug("%s: %s" % (k, v)) 72 | 73 | def process_response(self, silent=False): 74 | response = requests.get(self.request_url) 75 | json = response.json() 76 | if response.status_code >= 400: 77 | if not silent: 78 | log.error(json['message']) 79 | return {} 80 | return json 81 | 82 | def poll(self): 83 | log.info('Polling task at: %s' % self.request_url) 84 | json = self.process_response() 85 | # JSON could be set to None 86 | completed = json.get('ended', False) 87 | while not completed: 88 | sequence = ['.', '..', '...', '....'] 89 | for s in sequence: 90 | sys.stdout.write('\r' + ' '*80) 91 | string = "Waiting for completed task%s" % s 92 | sys.stdout.write('\r' + string) 93 | time.sleep(0.3) 94 | sys.stdout.flush() 95 | json = self.process_response(silent=True) 96 | completed = json.get('ended', False) 97 | sys.stdout.write('\r' + ' '*80) 98 | sys.stdout.flush() 99 | sys.stdout.write('\r'+'Task Completed!\n') 100 | for k, v in json.items(): 101 | log.debug("%s: %s" % (k, v)) 102 | 103 | def main(self): 104 | parser = Transport(self.arguments, options=self.options, check_help=True) 105 | parser.catch_help = self._help 106 | parser.parse_args() 107 | parser.catches_help() 108 | if not parser.unknown_commands: 109 | log.error("it is required to pass an identifer, but none was provided") 110 | raise SystemExit(1) 111 | self.identifier = parser.unknown_commands[-1] 112 | if parser.has('--poll'): 113 | return self.poll() 114 | 115 | for key in [ 116 | 'stdout', 'stderr', 'command', 'ended', 117 | 'started', 'succeeded', 'exit_code']: 118 | if parser.has(key): 119 | return self.get(key) 120 | 121 | # if nothing else matches, just try to give a generic, full summary 122 | self.summary() 123 | -------------------------------------------------------------------------------- /selinux/ceph_installer.te: -------------------------------------------------------------------------------- 1 | policy_module(ceph_installer, 1.0.0) 2 | require { 3 | type cert_t; 4 | type ssh_exec_t; 5 | type kernel_t; 6 | type ssh_port_t; 7 | type fs_t; 8 | type unreserved_port_t; 9 | type devpts_t; 10 | type ceph_installer_t; 11 | type krb5_conf_t; 12 | type ceph_installer_var_lib_t; 13 | type tmp_t; 14 | type devlog_t; 15 | type passwd_file_t; 16 | type ptmx_t; 17 | type net_conf_t; 18 | type file_context_t; 19 | type security_t; 20 | type syslogd_var_run_t; 21 | class key { write read view }; 22 | class unix_stream_socket connectto; 23 | class chr_file { write ioctl read open getattr }; 24 | class tcp_socket { name_bind name_connect }; 25 | class file { write execute read open getattr execute_no_trans }; 26 | class filesystem getattr; 27 | class sock_file { write link create unlink }; 28 | class security check_context; 29 | class unix_dgram_socket { create connect getopt sendto setopt }; 30 | class dir { search create rmdir }; 31 | class udp_socket { create connect getattr }; 32 | } 33 | 34 | ######################################## 35 | # 36 | # Declarations 37 | # 38 | 39 | type ceph_installer_t; 40 | type ceph_installer_exec_t; 41 | init_daemon_domain(ceph_installer_t, ceph_installer_exec_t) 42 | 43 | type ceph_installer_var_lib_t; 44 | files_type(ceph_installer_var_lib_t) 45 | 46 | type ceph_installer_tmp_t; 47 | files_tmp_file(ceph_installer_tmp_t) 48 | 49 | type ceph_installer_unit_file_t; 50 | systemd_unit_file(ceph_installer_unit_file_t) 51 | 52 | ######################################## 53 | # 54 | # ceph_installer local policy 55 | # 56 | allow ceph_installer_t self:fifo_file rw_fifo_file_perms; 57 | allow ceph_installer_t self:unix_stream_socket create_stream_socket_perms; 58 | allow ceph_installer_t self:tcp_socket create_stream_socket_perms; 59 | 60 | manage_dirs_pattern(ceph_installer_t, ceph_installer_var_lib_t, ceph_installer_var_lib_t) 61 | manage_files_pattern(ceph_installer_t, ceph_installer_var_lib_t, ceph_installer_var_lib_t) 62 | manage_lnk_files_pattern(ceph_installer_t, ceph_installer_var_lib_t, ceph_installer_var_lib_t) 63 | files_var_lib_filetrans(ceph_installer_t, ceph_installer_var_lib_t, { dir file lnk_file }) 64 | 65 | manage_files_pattern(ceph_installer_t, ceph_installer_tmp_t, ceph_installer_tmp_t) 66 | files_tmp_filetrans(ceph_installer_t, ceph_installer_tmp_t, { file }) 67 | 68 | kernel_read_system_state(ceph_installer_t) 69 | 70 | corecmd_exec_shell(ceph_installer_t) 71 | corecmd_exec_bin(ceph_installer_t) 72 | 73 | corenet_tcp_bind_generic_node(ceph_installer_t) 74 | corenet_tcp_connect_amqp_port(ceph_installer_t) 75 | #corenet_tcp_bind_intermapper_port(ceph_installer_t) 76 | 77 | libs_exec_ldconfig(ceph_installer_t) 78 | 79 | optional_policy(` 80 | apache_search_config(ceph_installer_t) 81 | ') 82 | 83 | permissive ceph_installer_t; 84 | 85 | allow ceph_installer_t ceph_installer_var_lib_t:sock_file { write create unlink link }; 86 | allow ceph_installer_t cert_t:dir search; 87 | allow ceph_installer_t cert_t:file { read getattr open }; 88 | allow ceph_installer_t devlog_t:sock_file write; 89 | allow ceph_installer_t devpts_t:filesystem getattr; 90 | allow ceph_installer_t file_context_t:dir search; 91 | allow ceph_installer_t file_context_t:file { read getattr open }; 92 | allow ceph_installer_t fs_t:filesystem getattr; 93 | allow ceph_installer_t kernel_t:unix_dgram_socket sendto; 94 | allow ceph_installer_t krb5_conf_t:file { read getattr open }; 95 | allow ceph_installer_t net_conf_t:file { read getattr open }; 96 | allow ceph_installer_t passwd_file_t:file { read getattr open }; 97 | allow ceph_installer_t ptmx_t:chr_file { read write ioctl open getattr }; 98 | allow ceph_installer_t security_t:file write; 99 | allow ceph_installer_t security_t:security check_context; 100 | allow ceph_installer_t self:key { write read view }; 101 | allow ceph_installer_t self:unix_dgram_socket { create connect getopt setopt }; 102 | allow ceph_installer_t ssh_exec_t:file { read execute open execute_no_trans }; 103 | allow ceph_installer_t ssh_port_t:tcp_socket name_connect; 104 | allow ceph_installer_t tmp_t:dir { create rmdir }; 105 | allow ceph_installer_t tmp_t:sock_file { write create unlink }; 106 | allow ceph_installer_t unreserved_port_t:tcp_socket name_bind; 107 | allow ceph_installer_t self:udp_socket { create connect getattr }; 108 | allow ceph_installer_t syslogd_var_run_t:sock_file write; 109 | 110 | ##!!!! This avc can be allowed using the boolean 'daemons_use_tty' 111 | allow ceph_installer_t devpts_t:chr_file open; 112 | 113 | ##!!!! This avc can be allowed using the boolean 'daemons_enable_cluster_mode' 114 | allow ceph_installer_t self:unix_stream_socket connectto; 115 | -------------------------------------------------------------------------------- /ceph_installer/hooks.py: -------------------------------------------------------------------------------- 1 | from celery.task.control import inspect 2 | from errno import errorcode 3 | from ceph_installer import models 4 | from ceph_installer.util import which 5 | from pecan.hooks import PecanHook 6 | from sqlalchemy.exc import OperationalError 7 | from webob.exc import WSGIHTTPException 8 | from webob.response import Response 9 | import logging 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class SystemCheckError(Exception): 16 | 17 | def __init__(self, message): 18 | self.message = message 19 | 20 | 21 | def ansible_exists(): 22 | """ 23 | Perform a simple check to see if ``ansibl-playbook`` executable is present 24 | in the system where the service is running. 25 | """ 26 | ansible_path = which('ansible-playbook') 27 | if not ansible_path: 28 | raise SystemCheckError('Could not find ansible in system paths') 29 | 30 | 31 | def celery_has_workers(): 32 | """ 33 | The ``stats()`` call will return different stats/metadata information about 34 | celery worker(s). An empty/None result will mean that there aren't any 35 | celery workers in use. 36 | """ 37 | stats = inspect().stats() 38 | if not stats: 39 | raise SystemCheckError('No running Celery worker was found') 40 | 41 | 42 | def rabbitmq_is_running(): 43 | """ 44 | If checking for worker stats, an ``IOError`` may be raised depending on the 45 | problem for the RabbitMQ connection. 46 | """ 47 | try: 48 | celery_has_workers() 49 | except IOError as e: 50 | msg = "Error connecting to RabbitMQ: " + str(e) 51 | if len(e.args): 52 | if errorcode.get(e.args[0]) == 'ECONNREFUSED': 53 | msg = "RabbitMQ is not running or not reachable" 54 | raise SystemCheckError(msg) 55 | 56 | 57 | def database_connection(): 58 | """ 59 | A very simple connection that should succeed if there is a good/correct 60 | database connection. 61 | """ 62 | try: 63 | models.Task.get(1) 64 | except OperationalError as exc: 65 | raise SystemCheckError( 66 | "Could not connect or retrieve information from the database: %s" % exc.message) 67 | 68 | 69 | system_checks = ( 70 | ansible_exists, 71 | rabbitmq_is_running, 72 | celery_has_workers, 73 | database_connection 74 | ) 75 | 76 | 77 | class CustomErrorHook(PecanHook): 78 | """ 79 | Ensure a traceback is logged correctly on error conditions. 80 | 81 | When an error condition is detected (one of the callables raises 82 | a ``SystemCheckError``) it sets the response status to 500 and returns 83 | a JSON response with the appropriate reason. 84 | """ 85 | 86 | def on_error(self, state, exc): 87 | if isinstance(exc, WSGIHTTPException): 88 | if exc.code == 404: 89 | logger.error("Not Found: %s" % state.request.url) 90 | return 91 | # explicit redirect codes that should not be handled at all by this 92 | # utility 93 | elif exc.code in [300, 301, 302, 303, 304, 305, 306, 307, 308]: 94 | return 95 | 96 | logger.exception('unhandled error by ceph-installer') 97 | for check in system_checks: 98 | try: 99 | check() 100 | except SystemCheckError as system_error: 101 | state.response.json_body = {'message': system_error.message} 102 | state.response.status = 500 103 | state.response.content_type = 'application/json' 104 | return state.response 105 | 106 | 107 | class JSONNonLocalRequest(WSGIHTTPException): 108 | """ 109 | WebOb doesn't allow setting the explicit content type 110 | when raising an HTTP exception. It forces the server to use plain text 111 | or HTML. We require a JSON response because we are validating JSON. 112 | 113 | We subclass form the base HTTP WebOb exception and force the response 114 | to be JSON. 115 | 116 | This class does not allow custom errors because its only purpose is to be 117 | used for the LocalHostWritesHook hook. 118 | """ 119 | 120 | code = 403 121 | title = 'Forbidden Request' 122 | explanation = 'Resource does not allow non-local requests' 123 | 124 | def generate_response(self, environ, start_response): 125 | if self.content_length is not None: 126 | del self.content_length 127 | headerlist = list(self.headerlist) 128 | content_type = 'application/json' 129 | body = '{"message": "this resource does not allow non-local requests"}' 130 | resp = Response( 131 | body, 132 | status=self.status, 133 | headerlist=headerlist, 134 | content_type=content_type 135 | ) 136 | resp.content_type = content_type 137 | return resp(environ, start_response) 138 | 139 | 140 | class LocalHostWritesHook(PecanHook): 141 | """ 142 | Allows all requests to go through that are HTTP GET, disallows all POST or 143 | DELETE requests that do not come from ``localhost`` 144 | """ 145 | 146 | def before(self, state): 147 | local_addresses = ['127.0.0.1', '127.1.1.0', 'localhost'] 148 | path_whitelist = ["/api/agent/"] 149 | if state.request.method in ['POST', 'DELETE', 'PUT']: 150 | if state.request.remote_addr not in local_addresses: 151 | if state.request.path not in path_whitelist: 152 | raise JSONNonLocalRequest() 153 | -------------------------------------------------------------------------------- /ceph_installer/tests/controllers/test_tasks.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from uuid import uuid4 3 | 4 | from ceph_installer.models import Task 5 | 6 | 7 | class TestTasksController(object): 8 | 9 | def create_task(self, **kw): 10 | Task( 11 | identifier=kw.get('identifier', str(uuid4())), 12 | endpoint='/api/rgw/', 13 | command='ansible-playbook -i "rgw.example.com," playbook.yml', 14 | stderr='', 15 | stdout='', 16 | started=kw.get('started', datetime.utcnow()), 17 | ended=kw.get('ended', datetime.utcnow()), 18 | succeeded=True 19 | ) 20 | 21 | def test_index_get_no_tasks(self, session): 22 | result = session.app.get("/api/tasks/") 23 | assert result.json == [] 24 | 25 | def test_index_get_single_task(self, session): 26 | self.create_task() 27 | session.commit() 28 | result = session.app.get("/api/tasks/") 29 | assert len(result.json) == 1 30 | 31 | def test_index_get_single_task_identifier(self, session): 32 | self.create_task(identifier='uuid-1') 33 | session.commit() 34 | result = session.app.get("/api/tasks/") 35 | assert result.json[0]['identifier'] == 'uuid-1' 36 | 37 | def test_index_get_single_task_endpoint(self, session): 38 | self.create_task() 39 | session.commit() 40 | result = session.app.get("/api/tasks/") 41 | assert result.json[0]['endpoint'] == '/api/rgw/' 42 | 43 | def test_index_get_single_task_command(self, session): 44 | self.create_task() 45 | session.commit() 46 | result = session.app.get("/api/tasks/") 47 | assert result.json[0]['command'] == 'ansible-playbook -i "rgw.example.com," playbook.yml' 48 | 49 | def test_index_get_single_task_stdout(self, session): 50 | self.create_task() 51 | session.commit() 52 | result = session.app.get("/api/tasks/") 53 | assert result.json[0]['stdout'] == '' 54 | 55 | def test_index_get_single_task_stderr(self, session): 56 | self.create_task() 57 | session.commit() 58 | result = session.app.get("/api/tasks/") 59 | assert result.json[0]['stderr'] == '' 60 | 61 | def test_index_get_single_task_started(self, session): 62 | started = datetime.utcnow() 63 | self.create_task(started=started) 64 | session.commit() 65 | result = session.app.get("/api/tasks/") 66 | assert result.json[0]['started'] == started.isoformat().replace('T', ' ') 67 | 68 | def test_index_get_single_task_ended(self, session): 69 | ended = datetime.utcnow() 70 | self.create_task(ended=ended) 71 | session.commit() 72 | result = session.app.get("/api/tasks/") 73 | assert result.json[0]['ended'] == ended.isoformat().replace('T', ' ') 74 | 75 | def test_index_get_single_task_succeeded(self, session): 76 | self.create_task() 77 | session.commit() 78 | result = session.app.get("/api/tasks/") 79 | assert result.json[0]['succeeded'] is True 80 | 81 | 82 | class TestTaskController(object): 83 | 84 | def create_task(self, **kw): 85 | Task( 86 | identifier=kw.get('identifier', str(uuid4())), 87 | endpoint='/api/rgw/', 88 | command='ansible-playbook -i "rgw.example.com," playbook.yml', 89 | stderr='', 90 | stdout='', 91 | started=kw.get('started', datetime.utcnow()), 92 | ended=kw.get('ended', datetime.utcnow()), 93 | succeeded=True 94 | ) 95 | 96 | def test_task_not_found(self, session): 97 | result = session.app.get( 98 | '/api/tasks/1234-asdf-1234-asdf/', 99 | expect_errors=True) 100 | assert result.status_int == 404 101 | 102 | def test_task_exists_with_metadata(self, session): 103 | identifier = '1234-asdf-1234-asdf' 104 | self.create_task(identifier=identifier) 105 | result = session.app.get('/api/tasks/1234-asdf-1234-asdf/') 106 | assert result.json['identifier'] 107 | 108 | 109 | class TestTaskControllerRequests(object): 110 | 111 | def create_task(self, **kw): 112 | 113 | Task( 114 | request=kw.get('request'), 115 | identifier=kw.get('identifier', '1234-asdf-1234-asdf'), 116 | endpoint='/api/rgw/', 117 | command='ansible-playbook -i "rgw.example.com," playbook.yml', 118 | stderr='', 119 | stdout='', 120 | started=kw.get('started', datetime.utcnow()), 121 | ended=kw.get('ended', datetime.utcnow()), 122 | succeeded=True 123 | ) 124 | 125 | def test_request_is_none(self, session): 126 | self.create_task() 127 | result = session.app.get( 128 | '/api/tasks/1234-asdf-1234-asdf/', 129 | expect_errors=True) 130 | print result.json 131 | assert result.json['user_agent'] == '' 132 | assert result.json['http_method'] == '' 133 | assert result.json['request'] == '' 134 | 135 | def test_request_with_valid_method(self, fake, session): 136 | fake_request = fake(method='POST') 137 | self.create_task(request=fake_request) 138 | result = session.app.get('/api/tasks/1234-asdf-1234-asdf/') 139 | assert result.json['http_method'] == 'POST' 140 | 141 | def test_request_with_valid_body(self, fake, session): 142 | fake_request = fake(body='{"host": "example.com"}') 143 | self.create_task(request=fake_request) 144 | result = session.app.get('/api/tasks/1234-asdf-1234-asdf/') 145 | assert result.json['request'] == '{"host": "example.com"}' 146 | 147 | def test_request_with_valid_user_agent(self, fake, session): 148 | fake_request = fake(user_agent='Mozilla/5.0') 149 | self.create_task(request=fake_request) 150 | result = session.app.get('/api/tasks/1234-asdf-1234-asdf/') 151 | assert result.json['user_agent'] == 'Mozilla/5.0' 152 | -------------------------------------------------------------------------------- /ceph_installer/schemas.py: -------------------------------------------------------------------------------- 1 | from notario.validators import types, recursive 2 | from notario.utils import forced_leaf_validator 3 | from notario.exceptions import Invalid 4 | from notario.decorators import optional 5 | 6 | 7 | def list_of_hosts(value): 8 | assert isinstance(value, list), "requires format: ['host1', 'host2']" 9 | 10 | 11 | def list_of_devices(value): 12 | assert isinstance(value, list), "requires format: ['/dev/sdb', '/dev/sdc']" 13 | 14 | 15 | @forced_leaf_validator 16 | def devices_object(_object, *args): 17 | error_msg = 'not of type dictionary or list' 18 | if isinstance(_object, dict): 19 | v = recursive.AllObjects((types.string, types.string)) 20 | # this is truly unfortunate but we don't have access to the 'tree' here 21 | # (the tree is the path to get to the failing key. We settle by just being 22 | # able to report nicely. 23 | v(_object, []) 24 | return 25 | 26 | try: 27 | assert isinstance(_object, list) 28 | except AssertionError: 29 | if args: 30 | raise Invalid('dict type', pair='value', msg=None, reason=error_msg, *args) 31 | raise 32 | 33 | 34 | 35 | def list_of_monitors(value): 36 | msg = 'requires format: [{"host": "mon1.host", "interface": "eth1"},{"host": "mon2.host", "address": "10.0.0.1"}]' 37 | assert isinstance(value, list), msg 38 | msg = 'address or interface is required for monitor lists: [{"host": "mon1", "interface": "eth1", {"host": "mon2", "address": "10.0.0.1"}]' 39 | for monitor in value: 40 | assert isinstance(monitor, dict), msg 41 | assert "host" in monitor, msg 42 | try: 43 | assert "interface" in monitor, msg 44 | except AssertionError: 45 | assert "address" in monitor, msg 46 | 47 | 48 | conf = ( 49 | (optional("global"), types.dictionary), 50 | (optional("mds"), types.dictionary), 51 | (optional("mon"), types.dictionary), 52 | (optional("osd"), types.dictionary), 53 | (optional("rgw"), types.dictionary), 54 | ) 55 | 56 | install_schema = ( 57 | ("hosts", list_of_hosts), 58 | (optional("redhat_storage"), types.boolean), 59 | (optional("redhat_use_cdn"), types.boolean), 60 | (optional("verbose"), types.boolean), 61 | ) 62 | 63 | agent_install_schema = ( 64 | ("hosts", list_of_hosts), 65 | (optional("master"), types.string), 66 | (optional("verbose"), types.boolean), 67 | ) 68 | 69 | mon_install_schema = ( 70 | (optional("calamari"), types.boolean), 71 | ("hosts", list_of_hosts), 72 | (optional("redhat_storage"), types.boolean), 73 | (optional("redhat_use_cdn"), types.boolean), 74 | (optional("verbose"), types.boolean), 75 | ) 76 | 77 | mon_configure_schema = ( 78 | (optional("address"), types.string), 79 | (optional("calamari"), types.boolean), 80 | (optional("cluster_name"), types.string), 81 | (optional("cluster_network"), types.string), 82 | (optional("conf"), conf), 83 | ("fsid", types.string), 84 | ("host", types.string), 85 | (optional("interface"), types.string), 86 | ("monitor_secret", types.string), 87 | (optional("monitors"), list_of_monitors), 88 | ("public_network", types.string), 89 | (optional("redhat_storage"), types.boolean), 90 | (optional("verbose"), types.boolean), 91 | ) 92 | 93 | osd_configure_schema = ( 94 | (optional("cluster_name"), types.string), 95 | (optional("cluster_network"), types.string), 96 | (optional("conf"), conf), 97 | ("devices", devices_object), 98 | ("fsid", types.string), 99 | ("host", types.string), 100 | ("journal_size", types.integer), 101 | ("monitors", list_of_monitors), 102 | ("public_network", types.string), 103 | (optional("redhat_storage"), types.boolean), 104 | (optional("verbose"), types.boolean), 105 | ) 106 | 107 | 108 | rgw_configure_schema = ( 109 | (optional("cluster_name"), types.string), 110 | (optional("cluster_network"), types.string), 111 | (optional("conf"), conf), 112 | (optional("email_address"), types.string), 113 | ("fsid", types.string), 114 | ("host", types.string), 115 | ("monitors", list_of_monitors), 116 | ("public_network", types.string), 117 | (optional("radosgw_civetweb_bind_ip"), types.string), 118 | (optional("radosgw_civetweb_num_threads"), types.integer), 119 | (optional("radosgw_civetweb_port"), types.integer), 120 | (optional("radosgw_dns_name"), types.string), 121 | (optional("radosgw_dns_s3website_name"), types.string), 122 | (optional("radosgw_keystone"), types.boolean), 123 | (optional("radosgw_keystone_accepted_roles"), types.array), 124 | (optional("radosgw_keystone_admin_domain"), types.string), 125 | (optional("radosgw_keystone_admin_password"), types.string), 126 | (optional("radosgw_keystone_admin_tenant"), types.string), 127 | (optional("radosgw_keystone_admin_token"), types.string), 128 | (optional("radosgw_keystone_admin_user"), types.string), 129 | (optional("radosgw_keystone_api_version"), types.integer), 130 | (optional("radosgw_keystone_auth_method"), types.string), 131 | (optional("radosgw_keystone_revocation_internal"), types.integer), 132 | (optional("radosgw_keystone_ssl"), types.boolean), 133 | (optional("radosgw_keystone_token_cache_size"), types.integer), 134 | (optional("radosgw_keystone_url"), types.string), 135 | (optional("radosgw_nss_db_path"), types.string), 136 | (optional("radosgw_resolve_cname"), types.boolean), 137 | (optional("radosgw_s3_auth_use_keystone"), types.boolean), 138 | (optional("radosgw_static_website"), types.boolean), 139 | (optional("radosgw_usage_log"), types.boolean), 140 | (optional("radosgw_usage_log_flush_threshold"), types.integer), 141 | (optional("radosgw_usage_log_tick_interval"), types.integer), 142 | (optional("radosgw_usage_max_shards"), types.integer), 143 | (optional("radosgw_usage_max_user_shards"), types.integer), 144 | (optional("redhat_storage"), types.boolean), 145 | (optional("redhat_use_cdn"), types.boolean), 146 | (optional("verbose"), types.boolean), 147 | ) 148 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Notario.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Notario.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Notario" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Notario" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | 2 | changelog 3 | ========= 4 | 5 | v1.3.1 (2017-07-14) 6 | ------------------- 7 | 8 | - Docs: update mon configure docs regarding monitor_secret. 9 | 10 | - Further test improvements. 11 | 12 | v1.3.0 (2017-04-11) 13 | ------------------- 14 | 15 | - Add RGW support. 16 | 17 | - Improve error message for missing /etc/os-release. 18 | 19 | - Avoid unicode errors when using the cli. 20 | 21 | - Improve tests to cover more scenarios. 22 | 23 | - Docs: further explaination of journal configurations. 24 | 25 | - Docs: fix rendering issue in /api/osd/configure docs. 26 | 27 | v1.2.2 (2017-01-27) 28 | ------------------- 29 | 30 | - Allows remote request to the /api/agent/ endpoint 31 | 32 | v1.2.1 (2017-01-24) 33 | ------------------- 34 | 35 | - Restricts POST, PUT and DELETE requests to local requests only. 36 | 37 | v1.2.0 (2017-01-17) 38 | ------------------- 39 | 40 | - Support for collocated journals when configuring OSDs 41 | 42 | v1.1.0 (2016-12-19) 43 | ------------------- 44 | 45 | - Support for v2.1.0 of ceph-ansible. 46 | 47 | - Many RPM packaging improvements. 48 | 49 | - Support for using systemd for dev deployments instead of circus. 50 | 51 | - Includes a functional testing scenario. 52 | 53 | v1.0.15 (2016-08-09) 54 | -------------------- 55 | 56 | - RPM packaging: Fix more SELinux AVC denials. 57 | 58 | - RPM packaging: Set ``ceph_installer_t`` SELinux domain to permissive. We have 59 | still found various AVC denials when running Skyring (Tendrl) with SELinux in 60 | enforcing mode. To make sure ceph-installer works while we fix the remaining 61 | issues with the policies, we are going to set the entire ceph_installer_t 62 | domain to permissive. 63 | 64 | With this change, the ceph-installer service still runs confined, but AVC 65 | denials will not prevent the service from operating. 66 | 67 | See https://lwn.net/Articles/303216/ for background on permissive SELinux 68 | domains. 69 | 70 | 71 | v1.0.14 (2016-07-13) 72 | -------------------- 73 | 74 | - RPM packaging: Fix SELinux AVC denial when setting up a cluster and SELinux 75 | is in enforcing mode. 76 | 77 | 78 | v1.0.13 (2016-07-12) 79 | -------------------- 80 | 81 | - RPM packaging: Add SELinux policy, and run pecan and celery processes 82 | confined. 83 | 84 | - Fix setup script shebang to be simply "#!/bin/bash". Prior to this change, 85 | executing the script via the shebang would lead to an error. 86 | 87 | - RPM packaging: don't allow Ansible 2 or above, since we've only tested with 88 | Ansible 1.9. 89 | 90 | 91 | v1.0.12 (2016-06-10) 92 | -------------------- 93 | 94 | - RPM packaging: silence output from the "yum install ceph-installer" command. 95 | 96 | - Build cleanups. 97 | 98 | 99 | v1.0.11 (2016-05-18) 100 | -------------------- 101 | 102 | - Sets CELERYD_CONCURRENCY=1 to ensure there is only one 103 | celery worker running. We need this to ensure tasks run 104 | sequentially. 105 | 106 | 107 | v1.0.10 (2016-05-11) 108 | -------------------- 109 | - Adds a man page for the ``ceph-installer`` cli 110 | 111 | - Now requires ceph-ansible >= 1.0.5 112 | 113 | v1.0.9 (2016-05-09) 114 | ------------------- 115 | - Create the necessary SSH keys on app startup. 116 | - Fix an issue where a subprocess couldn't communicate with stdin when using 117 | ``subprocess.Popen`` 118 | - No longer create SSH keys per request on ``/setup/key/`` 119 | 120 | 121 | v1.0.8 (2016-05-06) 122 | ------------------- 123 | - Include request information in tasks (available with JSON on ``/api/tasks/``) 124 | 125 | 126 | v1.0.7 (2016-05-04) 127 | ------------------- 128 | - The ``/setup/`` shell script now ensure that Python 2 is installed on 129 | Ubuntu distros. 130 | 131 | - Add additional server-side logging when ``ssh-keygen`` fails during the 132 | ``/setup/key/`` API call. 133 | 134 | - Several small doc updates: remove references to un-implemented callbacks, 135 | correct the type of ``cluster_name`` (it's a string). 136 | 137 | - Add trailing newline to the inventory files that ceph-installer passes 138 | internally to Ansible. This is not a user-visible change; it just makes it 139 | easier for developers to debug the auto-generated inventory files. 140 | 141 | 142 | v1.0.6 (2016-04-29) 143 | ------------------- 144 | 145 | - When the ``/setup/key/`` controller experiences a failure during the 146 | ``ssh-keygen`` operation, it will now return both the STDOUT and STDERR 147 | output from the failed key generation operation. Prior to this change, the 148 | controller would only return STDERR, and STDOUT was lost. The purpose of 149 | this change is to make it easier to debug when ``ssh-keygen`` fails. 150 | 151 | - The systemd units now cause the ``ceph-installer`` and 152 | ``ceph-installer-celery`` services to log both STDOUT and STDERR to the 153 | systemd journal. 154 | 155 | 156 | 1.0.5 (2016-04-19) 157 | ------------------ 158 | 159 | - Properly handle unicode output from ansible runs before storing them as 160 | a task in the database. 161 | 162 | - Prevent the same monitor from being duplicated in ceph.conf by removing it 163 | from ``monitors`` before calling ceph-ansible. 164 | 165 | 166 | 1.0.4 (2016-04-12) 167 | ------------------ 168 | 169 | - Fixes a bug that did not allow the use of the monitor ``address`` when 170 | configuring MONS or OSDs. 171 | 172 | 173 | 1.0.3 (2016-04-07) 174 | ------------------ 175 | 176 | - Adds the ability to provide a custom cluster name by using the ``cluster_name`` 177 | parameter when configuring MONs or OSDs. 178 | 179 | 180 | 1.0.2 (2016-03-28) 181 | ------------------ 182 | 183 | - Adds the ability to use ``address`` instead of ``interface`` when configuring 184 | MONs or OSDs. This replaces the ``monitor_interface`` parameter. 185 | 186 | 187 | 1.0.1 (2016-03-14) 188 | ------------------ 189 | 190 | - Fixes a bug where OSD configure fails when the OSD node being configured 191 | is also a MON 192 | 193 | - Allow values in ceph.conf to be set by using the ``conf`` parameter in the 194 | api/mon/configure/ and api/osd/configure/ endpoints 195 | 196 | - Adds the ability to set the ceph-installer address with the use of an 197 | environment varaible for the ceph-installer cli. 198 | 199 | 200 | 1.0.0 (2016-03-11) 201 | ------------------ 202 | 203 | - Initial stable release. 204 | -------------------------------------------------------------------------------- /docs/source/_themes/solarized.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | pygments.styles.solarized.light 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | The Solarized style, inspired by Schoonover. 7 | 8 | :copyright: Copyright 2012 by the Shoji KUMAGAI, see AUTHORS. 9 | :license: MIT, see LICENSE for details. 10 | """ 11 | 12 | from pygments.style import Style 13 | from pygments.token import Keyword, Name, Comment, String, Error, Text, \ 14 | Number, Operator, Generic, Whitespace, Other, Literal, Punctuation 15 | 16 | 17 | class LightStyle(Style): 18 | """ 19 | The Solarized Light style, inspired by Schoonover. 20 | """ 21 | background_color = '#fdf6e3' 22 | default_style = "" 23 | 24 | styles = { 25 | Text: '#657b83', # base00 ; class: '' 26 | Whitespace: '#fdf6e3', # base3 ; class: 'w' 27 | Error: '#dc322f', # red ; class: 'err' 28 | Other: '#657b83', # base00 ; class: 'x' 29 | 30 | Comment: 'italic #93a1a1', # base1 ; class: 'c' 31 | Comment.Multiline: 'italic #93a1a1', # base1 ; class: 'cm' 32 | Comment.Preproc: 'italic #93a1a1', # base1 ; class: 'cp' 33 | Comment.Single: 'italic #93a1a1', # base1 ; class: 'c1' 34 | Comment.Special: 'italic #93a1a1', # base1 ; class: 'cs' 35 | 36 | Keyword: '#859900', # green ; class: 'k' 37 | Keyword.Constant: '#859900', # green ; class: 'kc' 38 | Keyword.Declaration: '#859900', # green ; class: 'kd' 39 | Keyword.Namespace: '#cb4b16', # orange ; class: 'kn' 40 | Keyword.Pseudo: '#cb4b16', # orange ; class: 'kp' 41 | Keyword.Reserved: '#859900', # green ; class: 'kr' 42 | Keyword.Type: '#859900', # green ; class: 'kt' 43 | 44 | Operator: '#657b83', # base00 ; class: 'o' 45 | Operator.Word: '#859900', # green ; class: 'ow' 46 | 47 | Name: '#586e75', # base01 ; class: 'n' 48 | Name.Attribute: '#657b83', # base00 ; class: 'na' 49 | Name.Builtin: '#268bd2', # blue ; class: 'nb' 50 | Name.Builtin.Pseudo: 'bold #268bd2', # blue ; class: 'bp' 51 | Name.Class: '#268bd2', # blue ; class: 'nc' 52 | Name.Constant: '#b58900', # yellow ; class: 'no' 53 | Name.Decorator: '#cb4b16', # orange ; class: 'nd' 54 | Name.Entity: '#cb4b16', # orange ; class: 'ni' 55 | Name.Exception: '#cb4b16', # orange ; class: 'ne' 56 | Name.Function: '#268bd2', # blue ; class: 'nf' 57 | Name.Property: '#268bd2', # blue ; class: 'py' 58 | Name.Label: '#657b83', # base00 ; class: 'nc' 59 | Name.Namespace: '#b58900', # yellow ; class: 'nn' 60 | Name.Other: '#657b83', # base00 ; class: 'nx' 61 | Name.Tag: '#859900', # green ; class: 'nt' 62 | Name.Variable: '#cb4b16', # orange ; class: 'nv' 63 | Name.Variable.Class: '#268bd2', # blue ; class: 'vc' 64 | Name.Variable.Global: '#268bd2', # blue ; class: 'vg' 65 | Name.Variable.Instance: '#268bd2', # blue ; class: 'vi' 66 | 67 | Number: '#2aa198', # cyan ; class: 'm' 68 | Number.Float: '#2aa198', # cyan ; class: 'mf' 69 | Number.Hex: '#2aa198', # cyan ; class: 'mh' 70 | Number.Integer: '#2aa198', # cyan ; class: 'mi' 71 | Number.Integer.Long: '#2aa198', # cyan ; class: 'il' 72 | Number.Oct: '#2aa198', # cyan ; class: 'mo' 73 | 74 | Literal: '#657b83', # base00 ; class: 'l' 75 | Literal.Date: '#657b83', # base00 ; class: 'ld' 76 | 77 | Punctuation: '#657b83', # base00 ; class: 'p' 78 | 79 | String: '#2aa198', # cyan ; class: 's' 80 | String.Backtick: '#2aa198', # cyan ; class: 'sb' 81 | String.Char: '#2aa198', # cyan ; class: 'sc' 82 | String.Doc: '#2aa198', # cyan ; class: 'sd' 83 | String.Double: '#2aa198', # cyan ; class: 's2' 84 | String.Escape: '#cb4b16', # orange ; class: 'se' 85 | String.Heredoc: '#2aa198', # cyan ; class: 'sh' 86 | String.Interpol: '#cb4b16', # orange ; class: 'si' 87 | String.Other: '#2aa198', # cyan ; class: 'sx' 88 | String.Regex: '#2aa198', # cyan ; class: 'sr' 89 | String.Single: '#2aa198', # cyan ; class: 's1' 90 | String.Symbol: '#2aa198', # cyan ; class: 'ss' 91 | 92 | Generic: '#657b83', # base00 ; class: 'g' 93 | Generic.Deleted: '#657b83', # base00 ; class: 'gd' 94 | Generic.Emph: '#657b83', # base00 ; class: 'ge' 95 | Generic.Error: '#657b83', # base00 ; class: 'gr' 96 | Generic.Heading: '#657b83', # base00 ; class: 'gh' 97 | Generic.Inserted: '#657b83', # base00 ; class: 'gi' 98 | Generic.Output: '#657b83', # base00 ; class: 'go' 99 | Generic.Prompt: '#657b83', # base00 ; class: 'gp' 100 | Generic.Strong: '#657b83', # base00 ; class: 'gs' 101 | Generic.Subheading: '#657b83', # base00 ; class: 'gu' 102 | Generic.Traceback: '#657b83', # base00 ; class: 'gt' 103 | } 104 | -------------------------------------------------------------------------------- /docs/source/cluster.rst: -------------------------------------------------------------------------------- 1 | .. creating_a_cluster: 2 | 3 | Creating a cluster 4 | ================== 5 | To create a cluster there are a few steps involved. There is no single step to 6 | get everything done as many different requirements and restrictions are 7 | applied. Having granular steps allows the client to be able to handle any kind 8 | of logic as the process progresses. 9 | 10 | The following example assumes one monitor host ("mon.host") and two OSD hosts 11 | with one device each ("osd1.host" and "osd2.host"). It also assumes the 12 | installer ("installer.host") will be running on the same network and will be 13 | reachable over HTTP from and to the other nodes. 14 | 15 | Using callbacks is entirely optional and omitted from the examples below. 16 | Callbacks are an easier way to deal with asynchronous reports for requests and 17 | it is implemented with the ``"callback"`` key in most JSON POST requests. 18 | 19 | 1.- Bootstrap on each remote host (mon.host, osd1.host, and osd2.host):: 20 | 21 | wget http://installer.host/setup/ | sudo bash 22 | 23 | **Security**: the above example retrieves executable code over an 24 | unencrypted protocol and then executes it as root. You should only 25 | use this approach if you 100% trust your network and your DNS 26 | server. 27 | 28 | 29 | 2.- Install monitor: 30 | 31 | request:: 32 | 33 | curl -d '{"hosts": ["mon1.host"], "redhat_storage": true}' -X POST http://installer.hosts/api/mon/install/ 34 | 35 | response: 36 | 37 | .. sourcecode:: http 38 | 39 | HTTP/1.1 200 OK 40 | Content-Type: application/json 41 | 42 | { 43 | "endpoint": "/api/mon/install/", 44 | "succeeded": false, 45 | "stdout": null, 46 | "started": null, 47 | "exit_code": null, 48 | "ended": null, 49 | "command": null, 50 | "stderr": null, 51 | "identifier": "47f60562-a96b-4ac6-be07-71726b595793" 52 | } 53 | 54 | Note: the identifier can be immediately used get task metadata information:: 55 | 56 | curl http:://installer.host/api/tasks/47f60562-a96b-4ac6-be07-71726b595793/ 57 | 58 | response: 59 | 60 | .. sourcecode:: http 61 | 62 | HTTP/1.1 200 OK 63 | Content-Type: application/json 64 | 65 | { 66 | "endpoint": "/api/mon/install/", 67 | "succeeded": false, 68 | "stdout": null, 69 | "started": "2016-02-15 14:24:06.414728", 70 | "exit_code": null, 71 | "ended": null, 72 | "command": "/usr/local/bin/ansible-playbook /tmp/ceph-ansible/site.yml -i /var/folders/t8/smzdykh12h39f8r0vwv5vzf00000gn/T/47f60562-a96b-4ac6-be07-71726b595793__ilpiv --extra-vars {\"ceph_stable\": true} --tags package-install", 73 | "stderr": null, 74 | "identifier": "47f60562-a96b-4ac6-be07-71726b595793" 75 | } 76 | 77 | 3.- Install OSDs: 78 | 79 | request:: 80 | 81 | curl -d '{"hosts": ["osd1.host", "osd2.host"], "redhat_storage": true}' -X POST http://installer.hosts/api/osd/install/ 82 | 83 | response: 84 | 85 | .. sourcecode:: http 86 | 87 | HTTP/1.1 200 OK 88 | Content-Type: application/json 89 | 90 | 91 | { 92 | "endpoint": "/api/osd/install/", 93 | "succeeded": false, 94 | "stdout": null, 95 | "started": null, 96 | "exit_code": null, 97 | "ended": null, 98 | "command": null, 99 | "stderr": null, 100 | "identifier": "47f60562-a96b-4ac6-be07-71726b595793" 101 | } 102 | 103 | 104 | Task metadata for the previous request is then available at:: 105 | 106 | curl http:://installer.host/api/tasks/03965afd-6ae3-40e5-9530-3ac677a43226/ 107 | 108 | 109 | 4.- Configure monitor: 110 | 111 | request:: 112 | 113 | curl -d '{"host": "mon1.host", "monitor_interface": "eth0", "fsid": "deedcb4c-a67a-4997-93a6-92149ad2622a"}' -X POST http://installer.hosts/api/mon/configure/ 114 | 115 | response: 116 | 117 | .. sourcecode:: http 118 | 119 | HTTP/1.1 200 OK 120 | Content-Type: application/json 121 | 122 | { 123 | "endpoint": "/api/mon/configure/", 124 | "succeeded": false, 125 | "stdout": null, 126 | "started": null, 127 | "exit_code": null, 128 | "ended": null, 129 | "command": null, 130 | "stderr": null, 131 | "identifier": "4fe75438-1c76-40f9-b39c-9dbe78af28ed" 132 | } 133 | 134 | Task metadata for the previous request is then available at:: 135 | 136 | curl http:://installer.host/api/tasks/4fe75438-1c76-40f9-b39c-9dbe78af28ed/ 137 | 138 | 139 | 4.- Configure OSDs: 140 | Note that we are using ``journal_collocation`` flag to indicate we are going to 141 | collocate the journal in the same device as the OSD. This is *not ideal* and 142 | *not recommended for production use*, but it makes example setups easier to 143 | describe. 144 | 145 | request:: 146 | 147 | curl -d '{"host": "osd1.host", "devices": ["/dev/sdb/"], "journal_collocation": true, "fsid": "deedcb4c-a67a-4997-93a6-92149ad2622a"}' -X POST http://installer.hosts/api/osd/configure/ 148 | 149 | response: 150 | 151 | .. sourcecode:: http 152 | 153 | HTTP/1.1 200 OK 154 | Content-Type: application/json 155 | 156 | { 157 | "endpoint": "/api/osd/configure/", 158 | "succeeded": false, 159 | "stdout": null, 160 | "started": null, 161 | "exit_code": null, 162 | "ended": null, 163 | "command": null, 164 | "stderr": null, 165 | "identifier": "4af5189e-0e6c-4aa3-930c-b0ca6adb2545" 166 | } 167 | 168 | Task metadata for the previous request is then available at:: 169 | 170 | curl http:://installer.host/api/tasks/4af5189e-0e6c-4aa3-930c-b0ca6adb2545/ 171 | 172 | 173 | request:: 174 | 175 | curl -d '{"host": "osd2.host", "devices": ["/dev/sdc/"], 176 | "journal_collocation": true}' -X POST 177 | http://installer.hosts/api/osd/configure/ 178 | 179 | response: 180 | 181 | .. sourcecode:: http 182 | 183 | HTTP/1.1 200 OK 184 | Content-Type: application/json 185 | 186 | { 187 | "endpoint": "/api/osd/configure/", 188 | "succeeded": false, 189 | "stdout": null, 190 | "started": null, 191 | "exit_code": null, 192 | "ended": null, 193 | "command": null, 194 | "stderr": null, 195 | "identifier": "f248c190-4bb1-47d5-9188-c98434419f39" 196 | } 197 | 198 | Task metadata for the previous request is then available at:: 199 | 200 | curl http:://installer.host/api/tasks/f248c190-4bb1-47d5-9188-c98434419f39/ 201 | 202 | 203 | Once all tasks have completed correctly, the cluster should be up and in 204 | healthy state. 205 | 206 | 207 | -------------------------------------------------------------------------------- /ceph-installer.spec.in: -------------------------------------------------------------------------------- 1 | %global commit @COMMIT@ 2 | %global shortcommit %(c=%{commit}; echo ${c:0:7}) 3 | 4 | %define srcname ceph-installer 5 | 6 | %if 0%{?fedora} || 0%{?rhel} 7 | # get selinux policy version 8 | %{!?_selinux_policy_version: %global _selinux_policy_version %(sed -e 's,.*selinux-policy-\\([^/]*\\)/.*,\\1,' /usr/share/selinux/devel/policyhelp 2>/dev/null || echo 0.0.0)} 9 | %global selinux_types %(%{__awk} '/^#[[:space:]]*SELINUXTYPE=/,/^[^#]/ { if ($3 == "-") printf "%s ", $2 }' /etc/selinux/config 2>/dev/null) 10 | %global selinux_variants %([ -z "%{selinux_types}" ] && echo mls targeted || echo %{selinux_types}) 11 | %endif 12 | 13 | Name: ceph-installer 14 | Version: @VERSION@ 15 | Release: @RELEASE@%{?dist} 16 | Summary: A service to provision Ceph clusters 17 | License: MIT 18 | URL: https://github.com/ceph/ceph-installer 19 | Source0: %{name}-%{version}-%{shortcommit}.tar.gz 20 | 21 | BuildArch: noarch 22 | 23 | Requires: ansible >= 2.2.0.0 24 | Requires: ceph-ansible >= 2.0.0 25 | Requires: openssh 26 | Requires: python-celery 27 | Requires: python-gunicorn 28 | Requires: python-notario >= 0.0.11 29 | Requires: python-pecan >= 1 30 | Requires: python-pecan-notario 31 | Requires: python-requests 32 | Requires: python-sqlalchemy 33 | Requires: python-tambo 34 | Requires: rabbitmq-server 35 | Requires(pre): shadow-utils 36 | Requires(preun): systemd 37 | Requires(postun): systemd 38 | Requires(post): systemd 39 | 40 | %if 0%{?rhel} || 0%{?fedora} 41 | # SELinux deps 42 | Requires: policycoreutils, libselinux-utils 43 | Requires(post): selinux-policy >= %{_selinux_policy_version}, policycoreutils 44 | Requires(postun): policycoreutils 45 | BuildRequires: checkpolicy 46 | BuildRequires: selinux-policy-devel 47 | BuildRequires: /usr/share/selinux/devel/policyhelp 48 | BuildRequires: hardlink 49 | %endif 50 | 51 | BuildRequires: systemd 52 | BuildRequires: openssh 53 | BuildRequires: python2-devel 54 | BuildRequires: pytest 55 | BuildRequires: python-celery 56 | BuildRequires: python-docutils 57 | BuildRequires: python-pecan >= 1 58 | BuildRequires: python-pecan-notario 59 | BuildRequires: python-sqlalchemy 60 | 61 | %description 62 | An HTTP API to provision and control the deployment process of Ceph clusters. 63 | 64 | %prep 65 | %autosetup -p1 66 | 67 | %build 68 | %{__python} setup.py build 69 | %if 0%{?fedora} || 0%{?rhel} 70 | cd selinux 71 | for selinuxvariant in %{selinux_variants} 72 | do 73 | make NAME=${selinuxvariant} -f /usr/share/selinux/devel/Makefile 74 | mv ceph_installer.pp ceph_installer.pp.${selinuxvariant} 75 | make NAME=${selinuxvariant} -f /usr/share/selinux/devel/Makefile clean 76 | done 77 | cd - 78 | %endif 79 | 80 | %install 81 | %{__python} setup.py install -O1 --skip-build --root %{buildroot} 82 | 83 | install -p -D -m 644 systemd/ceph-installer.service \ 84 | %{buildroot}%{_unitdir}/ceph-installer.service 85 | 86 | install -p -D -m 644 systemd/ceph-installer-celery.service \ 87 | %{buildroot}%{_unitdir}/ceph-installer-celery.service 88 | 89 | install -p -D -m 644 systemd/ceph-installer.sysconfig \ 90 | %{buildroot}%{_sysconfdir}/sysconfig/ceph-installer 91 | 92 | install -p -D -m 644 systemd/80-ceph-installer.preset \ 93 | %{buildroot}%{_prefix}/lib/systemd/system-preset/80-ceph-installer.preset 94 | 95 | install -p -D -m 644 firewalld/ceph-installer.xml \ 96 | %{buildroot}%{_prefix}/lib/firewalld/services/ceph-installer.xml 97 | 98 | install -p -D -m 644 config/config.py \ 99 | %{buildroot}%{_sysconfdir}/ceph-installer/config.py 100 | 101 | mkdir -p %{buildroot}%{_var}/lib/ceph-installer 102 | 103 | mkdir -p %{buildroot}%{_mandir}/man8 104 | rst2man docs/source/man/index.rst > \ 105 | %{buildroot}%{_mandir}/man8/ceph-installer.8 106 | gzip %{buildroot}%{_mandir}/man8/ceph-installer.8 107 | %if 0%{?fedora} || 0%{?rhel} 108 | # Install SELinux policy 109 | for selinuxvariant in %{selinux_variants} 110 | do 111 | install -d %{buildroot}%{_datadir}/selinux/${selinuxvariant} 112 | install -p -m 644 selinux/ceph_installer.pp.${selinuxvariant} \ 113 | %{buildroot}%{_datadir}/selinux/${selinuxvariant}/ceph_installer.pp 114 | done 115 | /usr/sbin/hardlink -cv %{buildroot}%{_datadir}/selinux 116 | %endif 117 | 118 | %check 119 | py.test-%{python2_version} -v ceph_installer/tests 120 | 121 | %pre 122 | getent group ceph-installer >/dev/null || groupadd -r ceph-installer 123 | getent passwd ceph-installer >/dev/null || \ 124 | useradd -r -g ceph-installer -d %{_var}/lib/ceph-installer \ 125 | -s /bin/bash \ 126 | -c "system account for ceph-installer REST API" ceph-installer 127 | exit 0 128 | 129 | %post 130 | %if 0%{?fedora} || 0%{?rhel} 131 | ceph_installer_selinux() 132 | { 133 | # Set some SELinux booleans 134 | #setsebool httpd_can_network_connect=on 135 | #setsebool httpd_can_network_connect_db=on 136 | 137 | # Load the policy 138 | for selinuxvariant in %{selinux_variants} 139 | do 140 | /usr/sbin/semodule -s ${selinuxvariant} -i \ 141 | %{_datadir}/selinux/${selinuxvariant}/ceph_installer.pp &> /dev/null || : 142 | done 143 | # Relabel files 144 | /sbin/restorecon -Rv \ 145 | %{_bindir}/ceph-installer{,-gunicorn,-celery} \ 146 | %{_prefix}/lib/systemd/system-preset/80-ceph-installer.preset \ 147 | %{_prefix}/lib/systemd/system/ceph-installer{,-celery}.service \ 148 | %{_var}/lib/ceph-installer &> /dev/null || : 149 | } 150 | 151 | ceph_installer_selinux 152 | %endif 153 | if [ $1 -eq 1 ] ; then 154 | su - ceph-installer -c "/bin/pecan populate /etc/ceph-installer/config.py" &> /dev/null 155 | fi 156 | %systemd_post ceph-installer.service 157 | %systemd_post ceph-installer-celery.service 158 | systemctl start ceph-installer.service >/dev/null 2>&1 || : 159 | test -f %{_bindir}/firewall-cmd && firewall-cmd --reload --quiet || true 160 | 161 | %preun 162 | %systemd_preun ceph-installer.service 163 | %systemd_preun ceph-installer-celery.service 164 | 165 | %postun 166 | %systemd_postun_with_restart ceph-installer.service 167 | %systemd_postun_with_restart ceph-installer-celery.service 168 | %if 0%{?fedora} || 0%{?rhel} 169 | if [ $1 == 0 ] ; then 170 | for selinuxvariant in %{selinux_variants} 171 | do 172 | /usr/sbin/semodule -s ${selinuxvariant} -r ceph-installer &> /dev/null || : 173 | done 174 | fi 175 | %endif 176 | 177 | %files 178 | %doc README.rst 179 | %license LICENSE 180 | %{_bindir}/ceph-installer 181 | %{_bindir}/ceph-installer-celery 182 | %{_bindir}/ceph-installer-gunicorn 183 | %{python2_sitelib}/* 184 | %{_mandir}/man8/ceph-installer.8* 185 | %{_unitdir}/ceph-installer.service 186 | %{_unitdir}/ceph-installer-celery.service 187 | %{_prefix}/lib/systemd/system-preset/80-ceph-installer.preset 188 | %config(noreplace) %{_sysconfdir}/sysconfig/ceph-installer 189 | %config(noreplace) %{_sysconfdir}/ceph-installer/config.py 190 | %exclude %{_sysconfdir}/ceph-installer/config.pyc 191 | %exclude %{_sysconfdir}/ceph-installer/config.pyo 192 | %dir %attr (-, ceph-installer, ceph-installer) %{_var}/lib/ceph-installer 193 | %{_prefix}/lib/firewalld/services/ceph-installer.xml 194 | %if 0%{?fedora} || 0%{?rhel} 195 | %doc selinux/* 196 | %{_datadir}/selinux/*/ceph_installer.pp 197 | %endif 198 | 199 | %changelog 200 | -------------------------------------------------------------------------------- /ceph_installer/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | from pecan.testing import load_test_app 4 | 5 | from copy import deepcopy 6 | from pecan import conf 7 | from pecan import configuration 8 | from sqlalchemy import create_engine 9 | from sqlalchemy.pool import NullPool 10 | 11 | from ceph_installer import models as _db 12 | import pytest 13 | 14 | os.environ['HOME'] = tempfile.mkdtemp(suffix='.ceph-installer-home') 15 | 16 | DBNAME = 'ceph_installertest.db' 17 | BIND = 'sqlite:///' + os.environ['HOME'] 18 | 19 | 20 | def config_file(): 21 | here = os.path.abspath(os.path.dirname(__file__)) 22 | return os.path.join(here, 'config.py') 23 | 24 | 25 | @pytest.fixture 26 | def fake(): 27 | class Fake(object): 28 | def __init__(self, *a, **kw): 29 | for k, v, in kw.items(): 30 | setattr(self, k, v) 31 | return Fake 32 | 33 | 34 | @pytest.fixture 35 | def argtest(): 36 | """ 37 | Simple helper to use with monkeypatch so that a callable can be inspected 38 | afterwards to see if it was called with certain arguments. 39 | """ 40 | class TestArgs(object): 41 | def __call__(self, *args, **kwargs): 42 | self.args = list(args) 43 | self.kwargs = kwargs 44 | return TestArgs() 45 | 46 | 47 | @pytest.fixture(scope='session') 48 | def app(request): 49 | config = configuration.conf_from_file(config_file()).to_dict() 50 | 51 | # Add the appropriate connection string to the app config. 52 | config['sqlalchemy'] = { 53 | 'url': '%s/%s' % (BIND, DBNAME), 54 | 'echo': True, 55 | 'echo_pool': True, 56 | 'pool_recycle': 3600, 57 | 'encoding': 'utf-8' 58 | } 59 | 60 | # Set up a fake app 61 | app = TestApp(load_test_app(config)) 62 | return app 63 | 64 | 65 | @pytest.fixture(scope='session') 66 | def connection(app, request): 67 | """Session-wide test database.""" 68 | # Connect and create the temporary database 69 | print "=" * 80 70 | print "CREATING TEMPORARY DATABASE FOR TESTS" 71 | print "=" * 80 72 | 73 | # Bind and create the database tables 74 | _db.clear() 75 | engine_url = '%s/%s' % (BIND, DBNAME) 76 | 77 | db_engine = create_engine( 78 | engine_url, 79 | encoding='utf-8', 80 | poolclass=NullPool) 81 | 82 | # AKA models.start() 83 | _db.Session.bind = db_engine 84 | _db.metadata.bind = _db.Session.bind 85 | 86 | _db.Base.metadata.create_all(db_engine) 87 | _db.commit() 88 | _db.clear() 89 | 90 | #connection = db_engine.connect() 91 | 92 | def teardown(): 93 | _db.Base.metadata.drop_all(db_engine) 94 | 95 | request.addfinalizer(teardown) 96 | 97 | # Slap our test app on it 98 | _db.app = app 99 | return _db 100 | 101 | 102 | @pytest.fixture(scope='function') 103 | def session(connection, request): 104 | """Creates a new database session for a test.""" 105 | _config = configuration.conf_from_file(config_file()).to_dict() 106 | config = deepcopy(_config) 107 | 108 | # Add the appropriate connection string to the app config. 109 | config['sqlalchemy'] = { 110 | 'url': '%s/%s' % (BIND, DBNAME), 111 | 'encoding': 'utf-8', 112 | 'poolclass': NullPool 113 | } 114 | 115 | connection.start() 116 | 117 | def teardown(): 118 | from sqlalchemy.engine import reflection 119 | 120 | # Tear down and dispose the DB binding 121 | connection.clear() 122 | 123 | # start a transaction 124 | engine = conf.sqlalchemy.engine 125 | conn = engine.connect() 126 | trans = conn.begin() 127 | 128 | inspector = reflection.Inspector.from_engine(engine) 129 | 130 | # gather all data first before dropping anything. 131 | # some DBs lock after things have been dropped in 132 | # a transaction. 133 | conn.execute("DELETE FROM %s;" % ( 134 | ', '.join(inspector.get_table_names()) 135 | )) 136 | 137 | trans.commit() 138 | conn.close() 139 | 140 | request.addfinalizer(teardown) 141 | return connection 142 | 143 | 144 | class TestApp(object): 145 | """ 146 | A controller test starts a database transaction and creates a fake 147 | WSGI app. 148 | """ 149 | 150 | __headers__ = {} 151 | 152 | def __init__(self, app): 153 | self.app = app 154 | 155 | def _do_request(self, url, method='GET', **kwargs): 156 | methods = { 157 | 'GET': self.app.get, 158 | 'POST': self.app.post, 159 | 'POSTJ': self.app.post_json, 160 | 'PUT': self.app.put, 161 | 'DELETE': self.app.delete 162 | } 163 | kwargs.setdefault('headers', {}).update(self.__headers__) 164 | return methods.get(method, self.app.get)(str(url), **kwargs) 165 | 166 | def post_json(self, url, **kwargs): 167 | """ 168 | @param (string) url - The URL to emulate a POST request to 169 | @returns (paste.fixture.TestResponse) 170 | """ 171 | if 'extra_environ' not in kwargs: 172 | kwargs['extra_environ'] = dict(REMOTE_ADDR='127.0.0.1') 173 | return self._do_request(url, 'POSTJ', **kwargs) 174 | 175 | def post(self, url, **kwargs): 176 | """ 177 | @param (string) url - The URL to emulate a POST request to 178 | @returns (paste.fixture.TestResponse) 179 | """ 180 | if 'extra_environ' not in kwargs: 181 | kwargs['extra_environ'] = dict(REMOTE_ADDR='127.0.0.1') 182 | return self._do_request(url, 'POST', **kwargs) 183 | 184 | def get(self, url, **kwargs): 185 | """ 186 | @param (string) url - The URL to emulate a GET request to 187 | @returns (paste.fixture.TestResponse) 188 | """ 189 | return self._do_request(url, 'GET', **kwargs) 190 | 191 | def put(self, url, **kwargs): 192 | """ 193 | @param (string) url - The URL to emulate a PUT request to 194 | @returns (paste.fixture.TestResponse) 195 | """ 196 | if 'extra_environ' not in kwargs: 197 | kwargs['extra_environ'] = dict(REMOTE_ADDR='127.0.0.1') 198 | return self._do_request(url, 'PUT', **kwargs) 199 | 200 | def delete(self, url, **kwargs): 201 | """ 202 | @param (string) url - The URL to emulate a DELETE request to 203 | @returns (paste.fixture.TestResponse) 204 | """ 205 | if 'extra_environ' not in kwargs: 206 | kwargs['extra_environ'] = dict(REMOTE_ADDR='127.0.0.1') 207 | return self._do_request(url, 'DELETE', **kwargs) 208 | 209 | 210 | # this console logging configuration is basically just to be able to see output 211 | # in tests, and this file gets executed by py.test when it runs, so we get that 212 | # for free. 213 | import logging 214 | # Console Logger 215 | sh = logging.StreamHandler() 216 | sh.setLevel(logging.WARNING) 217 | 218 | formatter = logging.Formatter( 219 | fmt='%(asctime)s.%(msecs)03d %(process)d:%(levelname)s:%(name)s:%(message)s', 220 | datefmt='%Y-%m-%dT%H:%M:%S', 221 | ) 222 | sh.setFormatter(formatter) 223 | 224 | 225 | # because we're in a module already, __name__ is not the ancestor of 226 | # the rest of the package; use the root as the logger for everyone 227 | root_logger = logging.getLogger() 228 | 229 | # allow all levels at root_logger, handlers control individual levels 230 | root_logger.setLevel(logging.DEBUG) 231 | root_logger.addHandler(sh) 232 | 233 | console_loglevel = logging.DEBUG # start at DEBUG for now 234 | 235 | # Console Logger 236 | sh.setLevel(console_loglevel) 237 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # ceph_installer documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Jun 14 15:13:35 2012. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.append(os.path.abspath('_themes')) 20 | sys.path.append(os.path.abspath('..')) 21 | import ceph_installer 22 | 23 | # -- General configuration ----------------------------------------------------- 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be extensions 29 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 30 | extensions = ['sphinxcontrib.httpdomain'] 31 | http_index_localname = "ceph-installer HTTP API" 32 | 33 | # Add any paths that contain templates here, relative to this directory. 34 | templates_path = ['_templates'] 35 | 36 | # The suffix of source filenames. 37 | source_suffix = '.rst' 38 | 39 | # The encoding of source files. 40 | #source_encoding = 'utf-8-sig' 41 | 42 | # The master toctree document. 43 | master_doc = 'contents' 44 | 45 | # General information about the project. 46 | project = u'ceph-installer' 47 | copyright = u'2015-2016, Authors' 48 | 49 | # The version info for the project you're documenting, acts as replacement for 50 | # |version| and |release|, also used in various other places throughout the 51 | # built documents. 52 | # 53 | # The short X.Y version. 54 | version = ceph_installer.__version__ 55 | # The full version, including alpha/beta/rc tags. 56 | release = ceph_installer.__version__ 57 | 58 | # The language for content autogenerated by Sphinx. Refer to documentation 59 | # for a list of supported languages. 60 | #language = None 61 | 62 | # There are two options for replacing |today|: either, you set today to some 63 | # non-false value, then it is used: 64 | #today = '' 65 | # Else, today_fmt is used as the format for a strftime call. 66 | #today_fmt = '%B %d, %Y' 67 | 68 | # List of patterns, relative to source directory, that match files and 69 | # directories to ignore when looking for source files. 70 | exclude_patterns = [] 71 | 72 | # The reST default role (used for this markup: `text`) to use for all documents. 73 | #default_role = None 74 | 75 | # If true, '()' will be appended to :func: etc. cross-reference text. 76 | #add_function_parentheses = True 77 | 78 | # If true, the current module name will be prepended to all description 79 | # unit titles (such as .. function::). 80 | #add_module_names = True 81 | 82 | # If true, sectionauthor and moduleauthor directives will be shown in the 83 | # output. They are ignored by default. 84 | #show_authors = False 85 | 86 | # The name of the Pygments (syntax highlighting) style to use. 87 | pygments_style = 'solarized.LightStyle' 88 | 89 | # A list of ignored prefixes for module index sorting. 90 | #modindex_common_prefix = [] 91 | 92 | 93 | # -- Options for HTML output --------------------------------------------------- 94 | 95 | # The theme to use for HTML and HTML Help pages. See the documentation for 96 | # a list of builtin themes. 97 | #html_theme = '' 98 | 99 | # Theme options are theme-specific and customize the look and feel of a theme 100 | # further. For a list of options available for each theme, see the 101 | # documentation. 102 | #html_theme_options = { } 103 | 104 | # Add any paths that contain custom themes here, relative to this directory. 105 | html_theme_path = ['_themes'] 106 | 107 | # The name for this set of Sphinx documents. If None, it defaults to 108 | # " v documentation". 109 | html_title = 'ceph_installer' 110 | 111 | # A shorter title for the navigation bar. Default is the same as html_title. 112 | #html_short_title = None 113 | 114 | # The name of an image file (relative to this directory) to place at the top 115 | # of the sidebar. 116 | #html_logo = None 117 | 118 | # The name of an image file (within the static path) to use as favicon of the 119 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 120 | # pixels large. 121 | #html_favicon = None 122 | 123 | # Add any paths that contain custom static files (such as style sheets) here, 124 | # relative to this directory. They are copied after the builtin static files, 125 | # so a file named "default.css" will overwrite the builtin "default.css". 126 | html_static_path = ['_static'] 127 | 128 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 129 | # using the given strftime format. 130 | #html_last_updated_fmt = '%b %d, %Y' 131 | 132 | # If true, SmartyPants will be used to convert quotes and dashes to 133 | # typographically correct entities. 134 | #html_use_smartypants = True 135 | 136 | # Custom sidebar templates, maps document names to template names. 137 | #html_sidebars = {} 138 | 139 | # Additional templates that should be rendered to pages, maps page names to 140 | # template names. 141 | #html_additional_pages = {} 142 | 143 | # If false, no module index is generated. 144 | #html_domain_indices = True 145 | 146 | # If false, no index is generated. 147 | #html_use_index = True 148 | 149 | # If true, the index is split into individual pages for each letter. 150 | #html_split_index = False 151 | 152 | # If true, links to the reST sources are added to the pages. 153 | #html_show_sourcelink = True 154 | 155 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 156 | #html_show_sphinx = True 157 | 158 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 159 | #html_show_copyright = True 160 | 161 | # If true, an OpenSearch description file will be output, and all pages will 162 | # contain a tag referring to it. The value of this option must be the 163 | # base URL from which the finished HTML is served. 164 | #html_use_opensearch = '' 165 | 166 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 167 | #html_file_suffix = None 168 | 169 | # Output file base name for HTML help builder. 170 | htmlhelp_basename = 'Ceph-Installerdoc' 171 | 172 | 173 | # -- Options for LaTeX output -------------------------------------------------- 174 | 175 | latex_elements = { 176 | # The paper size ('letterpaper' or 'a4paper'). 177 | #'papersize': 'letterpaper', 178 | 179 | # The font size ('10pt', '11pt' or '12pt'). 180 | #'pointsize': '10pt', 181 | 182 | # Additional stuff for the LaTeX preamble. 183 | #'preamble': '', 184 | } 185 | 186 | # Grouping the document tree into LaTeX files. List of tuples 187 | # (source start file, target name, title, author, documentclass [howto/manual]). 188 | latex_documents = [ 189 | ('index', 'Ceph-Installer.tex', u'Ceph-Installer Documentation', 190 | u'Authors', 'manual'), 191 | ] 192 | 193 | # The name of an image file (relative to this directory) to place at the top of 194 | # the title page. 195 | #latex_logo = None 196 | 197 | # For "manual" documents, if this is true, then toplevel headings are parts, 198 | # not chapters. 199 | #latex_use_parts = False 200 | 201 | # If true, show page references after internal links. 202 | #latex_show_pagerefs = False 203 | 204 | # If true, show URL addresses after external links. 205 | #latex_show_urls = False 206 | 207 | # Documents to append as an appendix to all manuals. 208 | #latex_appendices = [] 209 | 210 | # If false, no module index is generated. 211 | #latex_domain_indices = True 212 | 213 | 214 | # -- Options for manual page output -------------------------------------------- 215 | 216 | # One entry per manual page. List of tuples 217 | # (source start file, name, description, authors, manual section). 218 | man_pages = [ 219 | ('man/index', 'ceph-installer', u'Ceph-Installer CLI Documentation', 220 | [u'Authors'], 1) 221 | ] 222 | 223 | # If true, show URL addresses after external links. 224 | #man_show_urls = False 225 | 226 | 227 | # -- Options for Texinfo output ------------------------------------------------ 228 | 229 | # Grouping the document tree into Texinfo files. List of tuples 230 | # (source start file, target name, title, author, 231 | # dir menu entry, description, category) 232 | texinfo_documents = [ 233 | ('index', 'Ceph-Installer', u'Ceph-Installer Documentation', 234 | u'Authors', 'Ceph-Installer', 'One line description of project.', 235 | 'Miscellaneous'), 236 | ] 237 | 238 | # Documents to append as an appendix to all manuals. 239 | #texinfo_appendices = [] 240 | 241 | # If false, no module index is generated. 242 | #texinfo_domain_indices = True 243 | 244 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 245 | #texinfo_show_urls = 'footnote' 246 | -------------------------------------------------------------------------------- /tests/functional/nightly-xenial/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | gather_facts: true 4 | 5 | - hosts: installer 6 | gather_facts: false 7 | vars: 8 | api_address: "http://localhost:8181/api" 9 | tasks: 10 | - name: install ceph on all mons 11 | uri: 12 | body_format: json 13 | method: POST 14 | url: "{{ api_address }}/mon/install" 15 | return_content: true 16 | body: 17 | hosts: "{{ groups['mons'] }}" 18 | register: mon_install 19 | 20 | - name: print output of mon install command 21 | debug: 22 | msg: "{{ mon_install.json }}" 23 | 24 | - name: wait for mon install to finish 25 | command: "ceph-installer task --poll {{ mon_install.json['identifier'] }}" 26 | no_log: true 27 | 28 | - name: get mon install status 29 | uri: 30 | method: GET 31 | url: "{{ api_address }}/tasks/{{ mon_install.json['identifier'] }}" 32 | return_content: true 33 | register: task_result 34 | no_log: true 35 | 36 | - fail: 37 | msg: "The mon install failed, see stdout: {{ task_result.json['stdout'] }}" 38 | when: not task_result.json["succeeded"] 39 | 40 | - name: install ceph on all osds 41 | uri: 42 | body_format: json 43 | method: POST 44 | url: "{{ api_address }}/osd/install" 45 | return_content: true 46 | body: 47 | hosts: "{{ groups['osds'] }}" 48 | register: osd_install 49 | 50 | - name: print output of osd install command 51 | debug: 52 | msg: "{{ osd_install.json }}" 53 | 54 | - name: wait for osd install to finish 55 | command: "ceph-installer task --poll {{ osd_install.json['identifier'] }}" 56 | no_log: true 57 | 58 | - name: get osd install status 59 | uri: 60 | method: GET 61 | url: "{{ api_address }}/tasks/{{ osd_install.json['identifier'] }}" 62 | return_content: true 63 | register: task_result 64 | no_log: true 65 | 66 | - fail: 67 | msg: "The osd install failed, see stdout: {{ task_result.json['stdout'] }}" 68 | when: not task_result.json["succeeded"] 69 | 70 | - name: install ceph on all rgws 71 | uri: 72 | body_format: json 73 | method: POST 74 | url: "{{ api_address }}/rgw/install" 75 | return_content: true 76 | body: 77 | hosts: "{{ groups['rgws'] }}" 78 | register: rgw_install 79 | 80 | - name: print output of rgw_install command 81 | debug: 82 | msg: "{{ rgw_install.json }}" 83 | 84 | - name: wait for rgw install to finish 85 | command: "ceph-installer task --poll {{ rgw_install.json['identifier'] }}" 86 | no_log: true 87 | 88 | - name: get rgw install status 89 | uri: 90 | method: GET 91 | url: "{{ api_address }}/tasks/{{ rgw_install.json['identifier'] }}" 92 | return_content: true 93 | register: task_result 94 | no_log: true 95 | 96 | - fail: 97 | msg: "The rgw install failed, see stdout: {{ task_result.json['stdout'] }}" 98 | when: not task_result.json["succeeded"] 99 | 100 | - name: configure ceph on first mon 101 | uri: 102 | body_format: json 103 | method: POST 104 | url: "{{ api_address }}/mon/configure" 105 | return_content: true 106 | body: 107 | host: "{{ groups['mons'][0] }}" 108 | interface: "{{ monitor_interface }}" 109 | fsid: "{{ fsid }}" 110 | monitor_secret: "{{ monitor_secret }}" 111 | public_network: "{{ public_network }}" 112 | register: mon_configure 113 | 114 | - name: print output of mon configure command 115 | debug: 116 | msg: "{{ mon_configure.json }}" 117 | 118 | - name: wait for mon configure to finish 119 | command: "ceph-installer task --poll {{ mon_configure.json['identifier'] }}" 120 | no_log: true 121 | 122 | - name: get mon configure status 123 | uri: 124 | method: GET 125 | url: "{{ api_address }}/tasks/{{ mon_configure.json['identifier'] }}" 126 | return_content: true 127 | register: task_result 128 | no_log: true 129 | 130 | - fail: 131 | msg: "The mon configure failed, see stdout: {{ task_result.json['stdout'] }}" 132 | when: not task_result.json["succeeded"] 133 | 134 | - name: configure ceph on second mon 135 | uri: 136 | body_format: json 137 | method: POST 138 | url: "{{ api_address }}/mon/configure" 139 | return_content: true 140 | body: 141 | host: "{{ groups['mons'][1] }}" 142 | interface: "{{ monitor_interface }}" 143 | fsid: "{{ fsid }}" 144 | monitor_secret: "{{ monitor_secret }}" 145 | public_network: "{{ public_network }}" 146 | monitors: 147 | - host: "{{ groups['mons'][0] }}" 148 | interface: "{{ monitor_interface }}" 149 | register: mon_configure 150 | 151 | - name: print output of mon configure command 152 | debug: 153 | msg: "{{ mon_configure.json }}" 154 | 155 | - name: wait for mon configure to finish 156 | command: "ceph-installer task --poll {{ mon_configure.json['identifier'] }}" 157 | no_log: true 158 | 159 | - name: get mon configure status 160 | uri: 161 | method: GET 162 | url: "{{ api_address }}/tasks/{{ mon_configure.json['identifier'] }}" 163 | return_content: true 164 | register: task_result 165 | no_log: true 166 | 167 | - fail: 168 | msg: "The mon configure failed, see stdout: {{ task_result.json['stdout'] }}" 169 | when: not task_result.json["succeeded"] 170 | 171 | - name: configure an OSD using a dedicated journal 172 | uri: 173 | body_format: json 174 | method: POST 175 | url: "{{ api_address }}/osd/configure" 176 | return_content: true 177 | body: 178 | host: "{{ groups['osds'][0] }}" 179 | fsid: "{{ fsid }}" 180 | public_network: "{{ public_network }}" 181 | monitors: "{{ monitors }}" 182 | journal_size: "{{ journal_size }}" 183 | devices: "{{ devices_dedicated_journal }}" 184 | register: osd_configure 185 | 186 | - name: print output of osd dedicated journal configure command 187 | debug: 188 | msg: "{{ osd_configure.json }}" 189 | 190 | - name: wait for osd configure to finish 191 | command: "ceph-installer task --poll {{ osd_configure.json['identifier'] }}" 192 | no_log: true 193 | 194 | - name: get osd configure status 195 | uri: 196 | method: GET 197 | url: "{{ api_address }}/tasks/{{ osd_configure.json['identifier'] }}" 198 | return_content: true 199 | register: task_result 200 | no_log: true 201 | 202 | - fail: 203 | msg: "The osd configure failed, see stdout: {{ task_result.json['stdout'] }}" 204 | when: not task_result.json["succeeded"] 205 | 206 | - name: configure an OSD using a collocated journal 207 | uri: 208 | body_format: json 209 | method: POST 210 | url: "{{ api_address }}/osd/configure" 211 | return_content: true 212 | body: 213 | host: "{{ groups['osds'][0] }}" 214 | fsid: "{{ fsid }}" 215 | public_network: "{{ public_network }}" 216 | monitors: "{{ monitors }}" 217 | journal_size: "{{ journal_size }}" 218 | devices: "{{ devices_collocated_journal }}" 219 | register: osd_configure 220 | 221 | - name: print output of osd collocated journal configure command 222 | debug: 223 | msg: "{{ osd_configure.json }}" 224 | 225 | - name: wait for osd configure to finish 226 | command: "ceph-installer task --poll {{ osd_configure.json['identifier'] }}" 227 | no_log: true 228 | 229 | - name: get osd configure status 230 | uri: 231 | method: GET 232 | url: "{{ api_address }}/tasks/{{ osd_configure.json['identifier'] }}" 233 | return_content: true 234 | register: task_result 235 | no_log: true 236 | 237 | - fail: 238 | msg: "The osd configure failed, see stdout: {{ task_result.json['stdout'] }}" 239 | when: not task_result.json["succeeded"] 240 | 241 | - name: configure ceph on all rgws 242 | uri: 243 | body_format: json 244 | method: POST 245 | url: "{{ api_address }}/rgw/configure" 246 | return_content: true 247 | body: 248 | host: "{{ groups['rgws'][0] }}" 249 | fsid: "{{ fsid }}" 250 | public_network: "{{ public_network }}" 251 | monitors: "{{ monitors }}" 252 | register: rgw_configure 253 | 254 | - name: print output of rgw configure command 255 | debug: 256 | msg: "{{ rgw_configure.json }}" 257 | 258 | - name: wait for rgw configure to finish 259 | command: "ceph-installer task --poll {{ rgw_configure.json['identifier'] }}" 260 | no_log: true 261 | 262 | - name: get rgw configure status 263 | uri: 264 | method: GET 265 | url: "{{ api_address }}/tasks/{{ rgw_configure.json['identifier'] }}" 266 | return_content: true 267 | register: task_result 268 | no_log: true 269 | 270 | - fail: 271 | msg: "The mon configure failed, see stdout: {{ task_result.json['stdout'] }}" 272 | when: not task_result.json["succeeded"] 273 | -------------------------------------------------------------------------------- /ceph_installer/tests/test_util.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from ceph_installer import util 5 | 6 | 7 | class TestGenerateInventoryFile(object): 8 | 9 | def test_single_host(self, tmpdir): 10 | result = util.generate_inventory_file([("mons", "google.com")], "uuid", tmp_dir=str(tmpdir)) 11 | with open(result, 'r') as f: 12 | data = [line.strip() for line in f.readlines()] 13 | assert "[mons]" in data 14 | assert "google.com" in data 15 | 16 | def test_correct_filename(self, tmpdir): 17 | result = util.generate_inventory_file([("mons", "google.com")], "uuid", tmp_dir=str(tmpdir)) 18 | assert "uuid_" in result 19 | 20 | def test_multiple_hosts(self, tmpdir): 21 | hosts = ['google.com', 'redhat.com'] 22 | result = util.generate_inventory_file([("mons", hosts)], "uuid", tmp_dir=str(tmpdir)) 23 | with open(result, 'r') as f: 24 | data = [line.strip() for line in f.readlines()] 25 | assert "[mons]" in data 26 | assert "google.com" in data 27 | assert "redhat.com" in data 28 | 29 | def test_multiple_groups(self, tmpdir): 30 | hosts = ['google.com', 'redhat.com'] 31 | inventory = [('mons', hosts), ('osds', 'osd1.host')] 32 | result = util.generate_inventory_file(inventory, "uuid", tmp_dir=str(tmpdir)) 33 | with open(result, 'r') as f: 34 | data = [line.strip() for line in f.readlines()] 35 | assert "[mons]" in data 36 | assert "google.com" in data 37 | assert "redhat.com" in data 38 | assert "[osds]" in data 39 | assert "osd1.host" in data 40 | 41 | def test_tmp_dir(self, tmpdir): 42 | result = util.generate_inventory_file([("mons", "google.com")], "uuid", tmp_dir=str(tmpdir)) 43 | assert str(tmpdir) in result 44 | 45 | 46 | class TestGetEndpoint(object): 47 | 48 | def test_no_args(self): 49 | result = util.get_endpoint('http://example.org/some/endpoint') 50 | assert result == 'http://example.org/' 51 | 52 | def test_one_arg(self): 53 | result = util.get_endpoint('http://example.org/some/endpoint', 'setup') 54 | assert result == 'http://example.org/setup/' 55 | 56 | def test_no_trailing_slash(self): 57 | result = util.get_endpoint('http://example.org', 'setup') 58 | assert result == 'http://example.org/setup/' 59 | 60 | 61 | class TestMkdir(object): 62 | 63 | def test_mkdir_success(self, tmpdir): 64 | path = os.path.join(str(tmpdir), 'mydir') 65 | util.mkdir(path) 66 | assert os.path.isdir(path) is True 67 | 68 | def test_mkdir_ignores_existing_dir(self, tmpdir): 69 | path = str(tmpdir) 70 | util.mkdir(path) 71 | assert os.path.isdir(path) is True 72 | 73 | def test_mkdir_does_not_ignore_existing_dir(self, tmpdir): 74 | path = str(tmpdir) 75 | with pytest.raises(OSError): 76 | util.mkdir(path, exist_ok=False) 77 | 78 | 79 | class TestGetInstallExtraVars(object): 80 | 81 | def test_no_extra_vars(self): 82 | data = dict() 83 | result = util.get_install_extra_vars(data) 84 | expected = { 85 | 'ceph_stable': True, 86 | 'fetch_directory': os.path.join(os.environ['HOME'], 'fetch'), 87 | } 88 | assert result == expected 89 | 90 | def test_redhat_storage_is_true(self): 91 | data = dict(redhat_storage=True) 92 | result = util.get_install_extra_vars(data) 93 | assert "ceph_stable_rh_storage" in result 94 | assert "ceph_stable_rh_storage_cdn_install" in result 95 | 96 | def test_redhat_storage_is_false(self): 97 | data = dict(redhat_storage=False) 98 | result = util.get_install_extra_vars(data) 99 | expected = { 100 | 'ceph_stable': True, 101 | 'fetch_directory': os.path.join(os.environ['HOME'], 'fetch'), 102 | } 103 | assert result == expected 104 | 105 | def test_redhat_use_cdn_is_true(self): 106 | data = dict(redhat_storage=True, redhat_use_cdn=True) 107 | result = util.get_install_extra_vars(data) 108 | assert "ceph_stable_rh_storage" in result 109 | assert "ceph_stable_rh_storage_cdn_install" in result 110 | 111 | def test_redhat_use_cdn_is_false(self): 112 | data = dict(redhat_storage=True, redhat_use_cdn=False) 113 | result = util.get_install_extra_vars(data) 114 | assert "ceph_stable_rh_storage" in result 115 | assert "ceph_stable_rh_storage_cdn_install" not in result 116 | 117 | def test_ceph_origin_exists_when_not_using_the_cdn(self): 118 | data = dict(redhat_storage=True, redhat_use_cdn=False) 119 | result = util.get_install_extra_vars(data) 120 | assert "ceph_origin" in result 121 | 122 | def test_ceph_origin_does_not_exists_when_using_the_cdn(self): 123 | data = dict(redhat_storage=True, redhat_use_cdn=True) 124 | result = util.get_install_extra_vars(data) 125 | assert "ceph_origin" not in result 126 | 127 | 128 | class TestGetOSDConfigureExtraVars(object): 129 | 130 | def setup(self): 131 | self.data = dict( 132 | host="node1", 133 | fsid="1720107309134", 134 | devices={'/dev/sdb': '/dev/sdc'}, 135 | monitors=[{"host": "mon1.host", "interface": "eth1"}], 136 | journal_size=100, 137 | public_network="0.0.0.0/24", 138 | ) 139 | 140 | def test_raw_multi_journal_is_set(self): 141 | result = util.get_osd_configure_extra_vars(self.data) 142 | assert "raw_multi_journal" in result 143 | 144 | def test_raw_journal_devices(self): 145 | result = util.get_osd_configure_extra_vars(self.data) 146 | assert "raw_journal_devices" in result 147 | assert result["raw_journal_devices"] == ["/dev/sdc"] 148 | 149 | def test_redhat_storage_not_present(self): 150 | data = self.data.copy() 151 | data["redhat_storage"] = True 152 | result = util.get_osd_configure_extra_vars(self.data) 153 | assert "redhat_storage" not in result 154 | 155 | def test_devices_should_be_a_list(self): 156 | # regression 157 | result = util.get_osd_configure_extra_vars(self.data) 158 | assert result["devices"] == ["/dev/sdb"] 159 | 160 | def test_monitor_name_is_set(self): 161 | # simulates the scenario where this host is a mon and an osd 162 | data = self.data.copy() 163 | data['host'] = "mon1.host" 164 | result = util.get_osd_configure_extra_vars(data) 165 | assert "monitor_name" in result 166 | assert result['monitor_name'] == "mon1.host" 167 | 168 | 169 | class TestGetOSDCollocatedConfigureExtraVars(object): 170 | 171 | def setup(self): 172 | self.data = dict( 173 | host="node1", 174 | fsid="1720107309134", 175 | devices=['/dev/sdb', '/dev/sdc'], 176 | monitors=[{"host": "mon1.host", "interface": "eth1"}], 177 | journal_size=100, 178 | public_network="0.0.0.0/24", 179 | ) 180 | 181 | def test_raw_multi_journal_is_not_set(self): 182 | result = util.get_osd_configure_extra_vars(self.data) 183 | assert "raw_multi_journal" not in result 184 | 185 | def test_journal_collocation_is_set(self): 186 | result = util.get_osd_configure_extra_vars(self.data) 187 | assert "journal_collocation" in result 188 | 189 | def test_collocated_devices(self): 190 | result = util.get_osd_configure_extra_vars(self.data) 191 | assert result["devices"] == ["/dev/sdb", "/dev/sdc"] 192 | assert 'raw_journal_devices' not in result 193 | 194 | def test_redhat_storage_not_present(self): 195 | data = self.data.copy() 196 | data["redhat_storage"] = True 197 | result = util.get_osd_configure_extra_vars(self.data) 198 | assert "redhat_storage" not in result 199 | 200 | def test_monitor_name_is_set(self): 201 | # simulates the scenario where this host is a mon and an osd 202 | data = self.data.copy() 203 | data['host'] = "mon1.host" 204 | result = util.get_osd_configure_extra_vars(data) 205 | assert "monitor_name" in result 206 | assert result['monitor_name'] == "mon1.host" 207 | 208 | 209 | class TestParseMonitors(object): 210 | 211 | def test_with_interface(self): 212 | data = [ 213 | {"host": "mon0.host", "interface": "eth0"}, 214 | {"host": "mon1.host", "interface": "eth1"}, 215 | ] 216 | results = util.parse_monitors(data) 217 | assert "mon0.host monitor_interface=eth0" in results 218 | assert "mon1.host monitor_interface=eth1" in results 219 | 220 | def test_with_address(self): 221 | data = [ 222 | {"host": "mon0.host", "address": "eth0"}, 223 | {"host": "mon1.host", "address": "eth1"}, 224 | ] 225 | results = util.parse_monitors(data) 226 | assert "mon0.host monitor_address=eth0" in results 227 | assert "mon1.host monitor_address=eth1" in results 228 | 229 | def test_one_with_no_interface(self): 230 | data = [ 231 | {"host": "mon0.host"}, 232 | {"host": "mon1.host", "interface": "eth1"}, 233 | ] 234 | results = util.parse_monitors(data) 235 | assert "mon0.host" == results[0] 236 | assert "mon1.host monitor_interface=eth1" in results 237 | 238 | def test_invalid_extra_var(self): 239 | data = [ 240 | {"host": "mon1.host", "foo": "bar"}, 241 | ] 242 | results = util.parse_monitors(data) 243 | assert "mon1.host" == results[0] 244 | 245 | 246 | class TestValidateMonitors(object): 247 | 248 | def test_host_given_as_monitor(self): 249 | data = [ 250 | {"host": "mon1.host", "foo": "bar"}, 251 | ] 252 | results = util.validate_monitors(data, "mon1.host") 253 | assert not results 254 | 255 | def test_host_not_given_as_monitor(self): 256 | data = [ 257 | {"host": "mon1.host", "foo": "bar"}, 258 | ] 259 | results = util.validate_monitors(data, "mon2.host") 260 | assert results == data 261 | 262 | def test_host_not_exact_match_in_monitors(self): 263 | data = [ 264 | {"host": "mon1.host", "foo": "bar"}, 265 | ] 266 | results = util.validate_monitors(data, "mon1") 267 | assert results == data 268 | 269 | def test_remove_host_from_monitors_leaving_others(self): 270 | data = [ 271 | {"host": "mon1.host", "foo": "bar"}, 272 | {"host": "mon2.host", "foo": "bar"}, 273 | ] 274 | results = util.validate_monitors(data, "mon1.host") 275 | assert len(results) == 1 276 | assert results[0]['host'] == "mon2.host" 277 | --------------------------------------------------------------------------------