├── os_apply_config ├── __init__.py ├── tests │ ├── __init__.py │ ├── templates │ │ └── etc │ │ │ ├── control │ │ │ ├── allow_empty │ │ │ ├── empty │ │ │ ├── empty.oac │ │ │ ├── mode │ │ │ ├── mode.oac │ │ │ └── allow_empty.oac │ │ │ ├── keystone │ │ │ └── keystone.conf │ │ │ └── glance │ │ │ └── script.conf │ ├── chown_templates │ │ ├── group.gid │ │ ├── group.name │ │ ├── owner.name │ │ ├── owner.uid │ │ ├── group.gid.oac │ │ ├── group.name.oac │ │ ├── owner.name.oac │ │ └── owner.uid.oac │ ├── test_json_renderer.py │ ├── test_oac_file.py │ ├── test_collect_config.py │ ├── test_value_type.py │ └── test_apply_config.py ├── config_exception.py ├── version.py ├── renderers.py ├── value_types.py ├── collect_config.py ├── oac_file.py └── apply_config.py ├── .stestr.conf ├── pyproject.toml ├── requirements.txt ├── .gitreview ├── releasenotes └── notes │ └── remove-py38-3c686984d115bccd.yaml ├── .coveragerc ├── zuul.d └── layout.yaml ├── test-requirements.txt ├── .gitignore ├── setup.py ├── tox.ini ├── setup.cfg ├── README.rst └── LICENSE /os_apply_config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /os_apply_config/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /os_apply_config/tests/templates/etc/control/allow_empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /os_apply_config/tests/templates/etc/control/empty: -------------------------------------------------------------------------------- 1 | foo 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/chown_templates/group.gid: -------------------------------------------------------------------------------- 1 | lorem gido 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/chown_templates/group.name: -------------------------------------------------------------------------------- 1 | namo gido 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/chown_templates/owner.name: -------------------------------------------------------------------------------- 1 | namo uido 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/chown_templates/owner.uid: -------------------------------------------------------------------------------- 1 | lorem uido 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/chown_templates/group.gid.oac: -------------------------------------------------------------------------------- 1 | group: 0 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/chown_templates/group.name.oac: -------------------------------------------------------------------------------- 1 | group: root 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/chown_templates/owner.name.oac: -------------------------------------------------------------------------------- 1 | owner: root 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/chown_templates/owner.uid.oac: -------------------------------------------------------------------------------- 1 | owner: 0 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/templates/etc/control/empty.oac: -------------------------------------------------------------------------------- 1 | # comment 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/templates/etc/control/mode: -------------------------------------------------------------------------------- 1 | lorem modus 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/templates/etc/control/mode.oac: -------------------------------------------------------------------------------- 1 | mode: 0755 2 | -------------------------------------------------------------------------------- /os_apply_config/tests/templates/etc/control/allow_empty.oac: -------------------------------------------------------------------------------- 1 | allow_empty: false 2 | -------------------------------------------------------------------------------- /.stestr.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_path=./os_apply_config/tests 3 | top_dir=./ 4 | 5 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["pbr>=6.1.1"] 3 | build-backend = "pbr.build" 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pbr>=2.0.0 # Apache-2.0 2 | 3 | pystache>=0.5.4 # MIT 4 | PyYAML>=3.12 # MIT 5 | -------------------------------------------------------------------------------- /os_apply_config/tests/templates/etc/keystone/keystone.conf: -------------------------------------------------------------------------------- 1 | [foo] 2 | database = {{database.url}} 3 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.opendev.org 3 | port=29418 4 | project=openstack/os-apply-config.git 5 | -------------------------------------------------------------------------------- /releasenotes/notes/remove-py38-3c686984d115bccd.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | upgrade: 3 | - | 4 | Python 3.8 is no longer supported. 5 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = os_apply_config 4 | omit = os_apply_config/tests/* 5 | 6 | [report] 7 | ignore_errors = True 8 | -------------------------------------------------------------------------------- /zuul.d/layout.yaml: -------------------------------------------------------------------------------- 1 | - project: 2 | templates: 3 | - check-requirements 4 | - openstack-cover-jobs 5 | - openstack-python3-jobs 6 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | hacking>=6.1.0,<6.2.0 # Apache-2.0 2 | 3 | coverage>=4.0 # Apache-2.0 4 | fixtures>=3.0.0 # Apache-2.0/BSD 5 | stestr>=2.0.0 # Apache-2.0 6 | testtools>=2.2.0 # MIT 7 | pyflakes>=2.2.0 8 | -------------------------------------------------------------------------------- /os_apply_config/tests/templates/etc/glance/script.conf: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import json 3 | import sys 4 | params = json.loads(sys.stdin.read()) 5 | x = params["x"] 6 | if x is None: raise Exception("undefined: x") 7 | print(x) 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg* 8 | dist 9 | build 10 | eggs 11 | parts 12 | bin 13 | var 14 | sdist 15 | develop-eggs 16 | .installed.cfg 17 | lib 18 | lib64 19 | 20 | # Installer logs 21 | pip-log.txt 22 | 23 | # Unit test / coverage reports 24 | .coverage 25 | cover 26 | .stestr/ 27 | .tox 28 | 29 | # Translations 30 | *.mo 31 | 32 | # Mr Developer 33 | .mr.developer.cfg 34 | .project 35 | .pydevproject 36 | 37 | # OpenStack Generated Files 38 | AUTHORS 39 | ChangeLog 40 | 41 | # Editors 42 | *~ 43 | *.swp 44 | -------------------------------------------------------------------------------- /os_apply_config/config_exception.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | class ConfigException(Exception): 18 | pass 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import setuptools 17 | 18 | 19 | setuptools.setup( 20 | setup_requires=['pbr>=2.0.0'], 21 | pbr=True) 22 | -------------------------------------------------------------------------------- /os_apply_config/version.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | 16 | import pbr.version 17 | 18 | version_info = pbr.version.VersionInfo('os-apply-config') 19 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py3,pep8 3 | minversion = 3.18.0 4 | 5 | [testenv] 6 | usedevelop = True 7 | deps = 8 | -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} 9 | -r{toxinidir}/requirements.txt 10 | -r{toxinidir}/test-requirements.txt 11 | 12 | [testenv:pep8] 13 | commands = flake8 14 | 15 | [testenv:cover] 16 | setenv = 17 | PYTHON=coverage run --source os_apply_config --parallel-mode 18 | commands = 19 | coverage erase 20 | stestr run {posargs} 21 | coverage combine 22 | coverage html -d cover 23 | coverage xml -o cover/coverage.xml 24 | coverage report 25 | 26 | [testenv:venv] 27 | commands = {posargs} 28 | 29 | [flake8] 30 | exclude = .venv,.tox,dist,doc,*.egg 31 | show-source = true 32 | # H904: Delay string interpolations at logging calls 33 | enable-extensions = H904 34 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = os-apply-config 3 | author = OpenStack 4 | author_email = openstack-discuss@lists.openstack.org 5 | summary = Config files from cloud metadata 6 | description_file = 7 | README.rst 8 | home_page = https://opendev.org/openstack/os-apply-config 9 | python_requires = >=3.9 10 | classifier = 11 | Development Status :: 4 - Beta 12 | Environment :: Console 13 | Environment :: OpenStack 14 | Intended Audience :: Developers 15 | Intended Audience :: Information Technology 16 | License :: OSI Approved :: Apache Software License 17 | Operating System :: OS Independent 18 | Programming Language :: Python 19 | Programming Language :: Python :: 3 20 | Programming Language :: Python :: 3 :: Only 21 | Programming Language :: Python :: 3.9 22 | Programming Language :: Python :: 3.10 23 | Programming Language :: Python :: 3.11 24 | Programming Language :: Python :: 3.12 25 | 26 | [files] 27 | packages = 28 | os_apply_config 29 | 30 | [entry_points] 31 | console_scripts = 32 | os-config-applier = os_apply_config.apply_config:main 33 | os-apply-config = os_apply_config.apply_config:main 34 | -------------------------------------------------------------------------------- /os_apply_config/tests/test_json_renderer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import json 17 | 18 | import testtools 19 | from testtools import content 20 | 21 | from os_apply_config import renderers 22 | 23 | TEST_JSON = '{"a":{"b":[1,2,3,"foo"],"c": "the quick brown fox"}}' 24 | 25 | 26 | class JsonRendererTestCase(testtools.TestCase): 27 | 28 | def test_json_renderer(self): 29 | context = json.loads(TEST_JSON) 30 | x = renderers.JsonRenderer() 31 | result = x.render('{{a.b}}', context) 32 | self.addDetail('result', content.text_content(result)) 33 | result_structure = json.loads(result) 34 | desire_structure = json.loads('[1,2,3,"foo"]') 35 | self.assertEqual(desire_structure, result_structure) 36 | result = x.render('{{a.c}}', context) 37 | self.addDetail('result', content.text_content(result)) 38 | self.assertEqual('the quick brown fox', result) 39 | -------------------------------------------------------------------------------- /os_apply_config/renderers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import json 17 | 18 | import pystache 19 | 20 | 21 | class JsonRenderer(pystache.Renderer): 22 | def __init__(self, 23 | file_encoding=None, 24 | string_encoding=None, 25 | decode_errors=None, 26 | search_dirs=None, 27 | file_extension=None, 28 | escape=None, 29 | partials=None, 30 | missing_tags=None): 31 | # json would be html escaped otherwise 32 | def escape_noop(u): 33 | return u 34 | if escape is None: 35 | escape = escape_noop 36 | return super().__init__(file_encoding, 37 | string_encoding, 38 | decode_errors, search_dirs, 39 | file_extension, escape, 40 | partials, missing_tags) 41 | 42 | def str_coerce(self, val): 43 | if val is None: 44 | return b'' 45 | return json.dumps(val) 46 | -------------------------------------------------------------------------------- /os_apply_config/value_types.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import re 17 | 18 | from os_apply_config import config_exception 19 | 20 | TYPES = { 21 | "int": "^[0-9]+$", 22 | "default": "^[A-Za-z0-9_]*$", 23 | "netaddress": "^[A-Za-z0-9/.:-]*$", 24 | "netdevice": "^[A-Za-z0-9/.:-]*$", 25 | "dsn": "(?#driver)^[a-zA-Z0-9]+://" 26 | "(?#username[:password])([a-zA-Z0-9+_-]+(:[^@]+)?)?" 27 | "(?#@host or file)(@?[a-zA-Z0-9/_.-]+)?" 28 | "(?#/dbname)(/[a-zA-Z0-9_-]+)?" 29 | "(?#?variable=value)(\\?[a-zA-Z0-9=_-]+)?$", 30 | "swiftdevices": "^(r\\d+z\\d+-[A-Za-z0-9.-_]+:%PORT%/[^,]+,?)+$", 31 | "username": "^[A-Za-z0-9_-]+$", 32 | "raw": "" 33 | } 34 | 35 | 36 | def ensure_type(string_value, type_name='default'): 37 | if type_name not in TYPES: 38 | raise ValueError( 39 | "requested validation of unknown type: %s" % type_name) 40 | if not re.match(TYPES[type_name], string_value): 41 | exception = config_exception.ConfigException 42 | raise exception("cannot interpret value '{}' as type {}".format( 43 | string_value, type_name)) 44 | return string_value 45 | -------------------------------------------------------------------------------- /os_apply_config/collect_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import copy 17 | import json 18 | import os 19 | 20 | from os_apply_config import config_exception as exc 21 | 22 | 23 | def read_configs(config_files): 24 | '''Generator yields data from any existing file in list config_files.''' 25 | for input_path in [x for x in config_files if x]: 26 | if os.path.exists(input_path): 27 | try: 28 | with open(input_path) as input_file: 29 | yield (input_file.read(), input_path) 30 | except OSError as e: 31 | raise exc.ConfigException('Could not open %s for reading. %s' % 32 | (input_path, e)) 33 | 34 | 35 | def parse_configs(config_data): 36 | '''Generator yields parsed json for each item passed in config_data.''' 37 | for input_data, input_path in config_data: 38 | try: 39 | yield json.loads(input_data) 40 | except ValueError: 41 | raise exc.ConfigException('Could not parse metadata file: %s' % 42 | input_path) 43 | 44 | 45 | def _deep_merge_dict(a, b): 46 | if not isinstance(b, dict): 47 | return b 48 | new_dict = copy.deepcopy(a) 49 | for k, v in iter(b.items()): 50 | if k in new_dict and isinstance(new_dict[k], dict): 51 | new_dict[k] = _deep_merge_dict(new_dict[k], v) 52 | else: 53 | new_dict[k] = copy.deepcopy(v) 54 | return new_dict 55 | 56 | 57 | def merge_configs(parsed_configs): 58 | '''Returns deep-merged dict from passed list of dicts.''' 59 | final_conf = {} 60 | for conf in parsed_configs: 61 | if conf and isinstance(conf, dict): 62 | final_conf = _deep_merge_dict(final_conf, conf) 63 | return final_conf 64 | 65 | 66 | def collect_config(os_config_files, fallback_paths=None): 67 | '''Convenience method to read, parse, and merge all paths.''' 68 | if fallback_paths: 69 | os_config_files = fallback_paths + os_config_files 70 | return merge_configs(parse_configs(read_configs(os_config_files))) 71 | -------------------------------------------------------------------------------- /os_apply_config/tests/test_oac_file.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import grp 17 | import pwd 18 | 19 | import testtools 20 | 21 | from os_apply_config import config_exception as exc 22 | from os_apply_config import oac_file 23 | 24 | 25 | class OacFileTestCase(testtools.TestCase): 26 | def test_mode_string(self): 27 | oacf = oac_file.OacFile('') 28 | mode = '0644' 29 | try: 30 | oacf.mode = mode 31 | except exc.ConfigException as e: 32 | self.assertIn("mode '%s' is not numeric" % mode, str(e)) 33 | 34 | def test_mode_range(self): 35 | oacf = oac_file.OacFile('') 36 | for mode in [-1, 0o1000]: 37 | try: 38 | oacf.mode = mode 39 | except exc.ConfigException as e: 40 | self.assertTrue("mode '%#o' out of range" % mode in str(e), 41 | "mode: %#o" % mode) 42 | 43 | for mode in [0, 0o777]: 44 | oacf.mode = mode 45 | 46 | def test_owner_positive(self): 47 | oacf = oac_file.OacFile('') 48 | users = pwd.getpwall() 49 | for name in [user[0] for user in users]: 50 | oacf.owner = name 51 | for uid in [user[2] for user in users]: 52 | oacf.owner = uid 53 | 54 | def test_owner_negative(self): 55 | oacf = oac_file.OacFile('') 56 | try: 57 | user = -1 58 | oacf.owner = user 59 | except exc.ConfigException as e: 60 | self.assertIn( 61 | "owner '%s' not found in passwd database" % user, str(e)) 62 | try: 63 | user = "za" 64 | oacf.owner = user 65 | except exc.ConfigException as e: 66 | self.assertIn( 67 | "owner '%s' not found in passwd database" % user, str(e)) 68 | 69 | def test_group_positive(self): 70 | oacf = oac_file.OacFile('') 71 | groups = grp.getgrall() 72 | for name in [group[0] for group in groups]: 73 | oacf.group = name 74 | for gid in [group[2] for group in groups]: 75 | oacf.group = gid 76 | 77 | def test_group_negative(self): 78 | oacf = oac_file.OacFile('') 79 | try: 80 | group = -1 81 | oacf.group = group 82 | except exc.ConfigException as e: 83 | self.assertIn( 84 | "group '%s' not found in group database" % group, str(e)) 85 | try: 86 | group = "za" 87 | oacf.group = group 88 | except exc.ConfigException as e: 89 | self.assertIn( 90 | "group '%s' not found in group database" % group, str(e)) 91 | -------------------------------------------------------------------------------- /os_apply_config/oac_file.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2014 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import grp 17 | import pwd 18 | 19 | from os_apply_config import config_exception as exc 20 | 21 | 22 | class OacFile: 23 | DEFAULTS = { 24 | 'allow_empty': True, 25 | 'mode': None, 26 | 'owner': None, 27 | 'group': None, 28 | } 29 | 30 | def __init__(self, body, **kwargs): 31 | super().__init__() 32 | self.body = body 33 | 34 | for k, v in self.DEFAULTS.items(): 35 | setattr(self, '_' + k, v) 36 | 37 | for k, v in kwargs.items(): 38 | if not hasattr(self, k): 39 | raise exc.ConfigException( 40 | "unrecognised file control key '%s'" % (k)) 41 | setattr(self, k, v) 42 | 43 | def __eq__(self, other): 44 | if type(other) is type(self): 45 | return self.__dict__ == other.__dict__ 46 | return False 47 | 48 | def __ne__(self, other): 49 | return not self.__eq__(other) 50 | 51 | def __repr__(self): 52 | a = ["OacFile(%s" % repr(self.body)] 53 | for key, default in self.DEFAULTS.items(): 54 | value = getattr(self, key) 55 | if value != default: 56 | a.append("{}={}".format(key, repr(value))) 57 | return ", ".join(a) + ")" 58 | 59 | def set(self, key, value): 60 | """Allows setting attrs as an expression rather than a statement.""" 61 | setattr(self, key, value) 62 | return self 63 | 64 | @property 65 | def allow_empty(self): 66 | """Returns allow_empty. 67 | 68 | If True and body='', no file will be created and any existing 69 | file will be deleted. 70 | """ 71 | return self._allow_empty 72 | 73 | @allow_empty.setter 74 | def allow_empty(self, value): 75 | if type(value) is not bool: 76 | raise exc.ConfigException( 77 | "allow_empty requires Boolean, got: '%s'" % value) 78 | self._allow_empty = value 79 | return self 80 | 81 | @property 82 | def mode(self): 83 | """The permissions to set on the file, EG 0755.""" 84 | return self._mode 85 | 86 | @mode.setter 87 | def mode(self, v): 88 | """Pass in the mode to set on the file. 89 | 90 | EG 0644. Must be between 0 and 0777, the sticky bit is not supported. 91 | """ 92 | if type(v) is not int: 93 | raise exc.ConfigException("mode '%s' is not numeric" % v) 94 | if not 0 <= v <= 0o777: 95 | raise exc.ConfigException("mode '%#o' out of range" % v) 96 | self._mode = v 97 | 98 | @property 99 | def owner(self): 100 | """The UID to set on the file, EG 'rabbitmq' or '501'.""" 101 | return self._owner 102 | 103 | @owner.setter 104 | def owner(self, v): 105 | """Pass in the UID to set on the file. 106 | 107 | EG 'rabbitmq' or 501. 108 | """ 109 | try: 110 | if type(v) is int: 111 | user = pwd.getpwuid(v) 112 | elif type(v) is str: 113 | user = pwd.getpwnam(v) 114 | else: 115 | raise exc.ConfigException( 116 | "owner '%s' must be a string or int" % v) 117 | except KeyError: 118 | raise exc.ConfigException( 119 | "owner '%s' not found in passwd database" % v) 120 | self._owner = user[2] 121 | 122 | @property 123 | def group(self): 124 | """The GID to set on the file, EG 'rabbitmq' or '501'.""" 125 | return self._group 126 | 127 | @group.setter 128 | def group(self, v): 129 | """Pass in the GID to set on the file. 130 | 131 | EG 'rabbitmq' or 501. 132 | """ 133 | try: 134 | if type(v) is int: 135 | group = grp.getgrgid(v) 136 | elif type(v) is str: 137 | group = grp.getgrnam(v) 138 | else: 139 | raise exc.ConfigException( 140 | "group '%s' must be a string or int" % v) 141 | except KeyError: 142 | raise exc.ConfigException( 143 | "group '%s' not found in group database" % v) 144 | self._group = group[2] 145 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | os-apply-config 3 | =============== 4 | 5 | .. image:: https://governance.openstack.org/tc/badges/os-apply-config.svg 6 | 7 | .. Change things from this point on 8 | 9 | ---------------------------------------------- 10 | Apply configuration from cloud metadata (JSON) 11 | ---------------------------------------------- 12 | 13 | What does it do? 14 | ================ 15 | 16 | It turns metadata from one or more JSON files like this:: 17 | 18 | {"keystone": {"database": {"host": "127.0.0.1", "user": "keystone", "password": "foobar"}}} 19 | 20 | into service config files like this:: 21 | 22 | [sql] 23 | connection = mysql://keystone:foobar@127.0.0.1/keystone 24 | ...other settings... 25 | 26 | Usage 27 | ===== 28 | 29 | Just pass it the path to a directory tree of templates:: 30 | 31 | sudo os-apply-config -t /home/me/my_templates 32 | 33 | By default it will read config files according to the contents of 34 | the file `/var/lib/os-collect-config/os_config_files.json`. In 35 | order to remain backward compatible it will also fall back to 36 | /var/run/os-collect-config/os_config_files.json, but the fallback 37 | path is deprecated and will be removed in a later release. The main 38 | path can be changed with the command line switch `--os-config-files`, 39 | or the environment variable `OS_CONFIG_FILES_PATH`. The list can 40 | also be overridden with the environment variable `OS_CONFIG_FILES`. 41 | If overriding with `OS_CONFIG_FILES`, the paths are expected to be colon, 42 | ":", separated. Each json file referred to must have a mapping as their 43 | root structure. Keys in files mentioned later in the list will override 44 | keys in earlier files from this list. For example:: 45 | 46 | OS_CONFIG_FILES=/tmp/ec2.json:/tmp/cfn.json os-apply-config 47 | 48 | This will read `ec2.json` and `cfn.json`, and if they have any 49 | overlapping keys, the value from `cfn.json` will be used. That will 50 | populate the tree for any templates found in the template path. See 51 | https://opendev.org/openstack/os-collect-config for a 52 | program that will automatically collect data and populate this list. 53 | 54 | You can also override `OS_CONFIG_FILES` with the `--metadata` command 55 | line option, specifying it multiple times instead of colon separating 56 | the list. 57 | 58 | `os-apply-config` will also always try to read metadata in the old 59 | legacy paths first to populate the tree. These paths can be changed 60 | with `--fallback-metadata`. 61 | 62 | Templates 63 | ========= 64 | 65 | The template directory structure should mimic a root filesystem, and 66 | contain templates for only those files you want configured. For 67 | example:: 68 | 69 | ~/my_templates$ tree 70 | . 71 | +-- etc 72 | +-- keystone 73 | | +-- keystone.conf 74 | +-- mysql 75 | +-- mysql.conf 76 | 77 | An example tree can be found `here `_. 78 | 79 | If a template is executable it will be treated as an *executable 80 | template*. Otherwise, it will be treated as a *mustache template*. 81 | 82 | Mustache Templates 83 | ------------------ 84 | 85 | If you don't need any logic, just some string substitution, use a 86 | mustache template. 87 | 88 | Metadata settings are accessed with dot ('.') notation:: 89 | 90 | [sql] 91 | connection = mysql://{{keystone.database.user}}:{{keystone.database.password}}@{{keystone.database.host}}/keystone 92 | 93 | Executable Templates 94 | -------------------- 95 | 96 | Configuration requiring logic is expressed in executable templates. 97 | 98 | An executable template is a script which accepts configuration as a 99 | JSON string on standard in, and writes a config file to standard out. 100 | 101 | The script should exit non-zero if it encounters a problem, so that 102 | os-apply-config knows what's up. 103 | 104 | The output of the script will be written to the path corresponding to 105 | the executable template's path in the template tree:: 106 | 107 | #!/usr/bin/env ruby 108 | require 'json' 109 | params = JSON.parse STDIN.read 110 | puts "connection = mysql://#{c['keystone']['database']['user']}:#{c['keystone']['database']['password']}@#{c['keystone']['database']['host']}/keystone" 111 | 112 | You could even embed mustache in a heredoc, and use that:: 113 | 114 | #!/usr/bin/env ruby 115 | require 'json' 116 | require 'mustache' 117 | params = JSON.parse STDIN.read 118 | 119 | template = <<-eos 120 | [sql] 121 | connection = mysql://{{keystone.database.user}}:{{keystone.database.password}}@{{keystone.database.host}}/keystone 122 | 123 | [log] 124 | ... 125 | eos 126 | 127 | # tweak params here... 128 | 129 | puts Mustache.render(template, params) 130 | 131 | 132 | Quick Start 133 | =========== 134 | :: 135 | 136 | # install it 137 | sudo pip install -U git+https://opendev.org/openstack/os-apply-config.git 138 | 139 | # grab example templates 140 | git clone https://opendev.org/openstack/tripleo-image-elements /tmp/config 141 | 142 | # run it 143 | os-apply-config -t /tmp/config/elements/nova/os-apply-config/ -m /tmp/config/elements/seed-stack-config/config.json -o /tmp/config_output 144 | -------------------------------------------------------------------------------- /os_apply_config/tests/test_collect_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import json 17 | import os 18 | 19 | import fixtures 20 | import testtools 21 | 22 | from os_apply_config import collect_config 23 | from os_apply_config import config_exception as exc 24 | 25 | 26 | class OCCTestCase(testtools.TestCase): 27 | def test_collect_config(self): 28 | conflict_configs = [('ec2', {'local-ipv4': '192.0.2.99', 29 | 'instance-id': 'feeddead'}), 30 | ('cfn', {'foo': {'bar': 'foo-bar'}, 31 | 'local-ipv4': '198.51.100.50'})] 32 | config_files = [] 33 | tdir = self.useFixture(fixtures.TempDir()) 34 | for name, config in conflict_configs: 35 | path = os.path.join(tdir.path, '%s.json' % name) 36 | with open(path, 'w') as out: 37 | out.write(json.dumps(config)) 38 | config_files.append(path) 39 | config = collect_config.collect_config(config_files) 40 | self.assertEqual( 41 | {'local-ipv4': '198.51.100.50', 42 | 'instance-id': 'feeddead', 43 | 'foo': {'bar': 'foo-bar'}}, config) 44 | 45 | def test_collect_config_fallback(self): 46 | tdir = self.useFixture(fixtures.TempDir()) 47 | with open(os.path.join(tdir.path, 'does_exist.json'), 'w') as t: 48 | t.write(json.dumps({'a': 1})) 49 | noexist_path = os.path.join(tdir.path, 'does_not_exist.json') 50 | 51 | config = collect_config.collect_config([], [noexist_path, t.name]) 52 | self.assertEqual({'a': 1}, config) 53 | 54 | with open(os.path.join(tdir.path, 'does_exist_new.json'), 'w') as t2: 55 | t2.write(json.dumps({'a': 2})) 56 | 57 | config = collect_config.collect_config([t2.name], [t.name]) 58 | self.assertEqual({'a': 2}, config) 59 | 60 | config = collect_config.collect_config([], [t.name, noexist_path]) 61 | self.assertEqual({'a': 1}, config) 62 | self.assertEqual({}, 63 | collect_config.collect_config([], [noexist_path])) 64 | self.assertEqual({}, 65 | collect_config.collect_config([])) 66 | 67 | def test_failed_read(self): 68 | tdir = self.useFixture(fixtures.TempDir()) 69 | unreadable_path = os.path.join(tdir.path, 'unreadable.json') 70 | with open(unreadable_path, 'w') as u: 71 | u.write(json.dumps({})) 72 | os.chmod(unreadable_path, 0o000) 73 | self.assertRaises( 74 | exc.ConfigException, 75 | lambda: list(collect_config.read_configs([unreadable_path]))) 76 | 77 | def test_bad_json(self): 78 | tdir = self.useFixture(fixtures.TempDir()) 79 | bad_json_path = os.path.join(tdir.path, 'bad.json') 80 | self.assertRaises( 81 | exc.ConfigException, 82 | lambda: list(collect_config.parse_configs([('{', bad_json_path)]))) 83 | 84 | 85 | class TestMergeConfigs(testtools.TestCase): 86 | 87 | def test_merge_configs_noconflict(self): 88 | noconflict_configs = [{'a': '1'}, 89 | {'b': 'Y'}] 90 | result = collect_config.merge_configs(noconflict_configs) 91 | self.assertEqual({'a': '1', 92 | 'b': 'Y'}, result) 93 | 94 | def test_merge_configs_conflict(self): 95 | conflict_configs = [{'a': '1'}, {'a': 'Z'}] 96 | result = collect_config.merge_configs(conflict_configs) 97 | self.assertEqual({'a': 'Z'}, result) 98 | 99 | def test_merge_configs_deep_conflict(self): 100 | deepconflict_conf = [{'a': '1'}, 101 | {'b': {'x': 'foo-bar', 'y': 'tribbles'}}, 102 | {'b': {'x': 'shazam'}}] 103 | result = collect_config.merge_configs(deepconflict_conf) 104 | self.assertEqual({'a': '1', 105 | 'b': {'x': 'shazam', 'y': 'tribbles'}}, result) 106 | 107 | def test_merge_configs_type_conflict(self): 108 | type_conflict = [{'a': 1}, {'a': [7, 8, 9]}] 109 | result = collect_config.merge_configs(type_conflict) 110 | self.assertEqual({'a': [7, 8, 9]}, result) 111 | 112 | def test_merge_configs_nested_type_conflict(self): 113 | type_conflict = [{'a': {'foo': 'bar'}}, {'a': 'shazam'}] 114 | result = collect_config.merge_configs(type_conflict) 115 | self.assertEqual({'a': 'shazam'}, result) 116 | 117 | def test_merge_configs_list_conflict(self): 118 | list_conflict = [{'a': [1, 2, 3]}, 119 | {'a': [4, 5, 6]}] 120 | result = collect_config.merge_configs(list_conflict) 121 | self.assertEqual({'a': [4, 5, 6]}, result) 122 | 123 | def test_merge_configs_empty_notdict(self): 124 | list_conflict = [[], {'a': '1'}, '', None, 'tacocat', 125 | {'b': '2'}, {}, 'baseball'] 126 | result = collect_config.merge_configs(list_conflict) 127 | self.assertEqual({'a': '1', 'b': '2'}, result) 128 | -------------------------------------------------------------------------------- /os_apply_config/tests/test_value_type.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import testtools 17 | 18 | from os_apply_config import config_exception 19 | from os_apply_config import value_types 20 | 21 | 22 | class ValueTypeTestCase(testtools.TestCase): 23 | 24 | def test_unknown_type(self): 25 | self.assertRaises( 26 | ValueError, value_types.ensure_type, "foo", "badtype") 27 | 28 | def test_int(self): 29 | self.assertEqual("123", value_types.ensure_type("123", "int")) 30 | 31 | def test_default(self): 32 | self.assertEqual("foobar", 33 | value_types.ensure_type("foobar", "default")) 34 | self.assertEqual("x86_64", 35 | value_types.ensure_type("x86_64", "default")) 36 | 37 | def test_default_bad(self): 38 | self.assertRaises(config_exception.ConfigException, 39 | value_types.ensure_type, "foo\nbar", "default") 40 | 41 | def test_default_empty(self): 42 | self.assertEqual('', 43 | value_types.ensure_type('', 'default')) 44 | 45 | def test_raw_empty(self): 46 | self.assertEqual('', 47 | value_types.ensure_type('', 'raw')) 48 | 49 | def test_net_address_ipv4(self): 50 | self.assertEqual('192.0.2.1', value_types.ensure_type('192.0.2.1', 51 | 'netaddress')) 52 | 53 | def test_net_address_cidr(self): 54 | self.assertEqual('192.0.2.0/24', 55 | value_types.ensure_type('192.0.2.0/24', 'netaddress')) 56 | 57 | def test_ent_address_ipv6(self): 58 | self.assertEqual('::', value_types.ensure_type('::', 'netaddress')) 59 | self.assertEqual('2001:db8::2:1', value_types.ensure_type( 60 | '2001:db8::2:1', 'netaddress')) 61 | 62 | def test_net_address_dns(self): 63 | self.assertEqual('host.0domain-name.test', 64 | value_types.ensure_type('host.0domain-name.test', 65 | 'netaddress')) 66 | 67 | def test_net_address_empty(self): 68 | self.assertEqual('', value_types.ensure_type('', 'netaddress')) 69 | 70 | def test_net_address_bad(self): 71 | self.assertRaises(config_exception.ConfigException, 72 | value_types.ensure_type, "192.0.2.1;DROP TABLE foo", 73 | 'netaddress') 74 | 75 | def test_netdevice(self): 76 | self.assertEqual('eth0', 77 | value_types.ensure_type('eth0', 'netdevice')) 78 | 79 | def test_netdevice_dash(self): 80 | self.assertEqual('br-ctlplane', 81 | value_types.ensure_type('br-ctlplane', 'netdevice')) 82 | 83 | def test_netdevice_alias(self): 84 | self.assertEqual('eth0:1', 85 | value_types.ensure_type('eth0:1', 'netdevice')) 86 | 87 | def test_netdevice_bad(self): 88 | self.assertRaises(config_exception.ConfigException, 89 | value_types.ensure_type, "br-tun; DROP TABLE bar", 90 | 'netdevice') 91 | 92 | def test_dsn_nopass(self): 93 | test_dsn = 'mysql://user@host/db' 94 | self.assertEqual(test_dsn, value_types.ensure_type(test_dsn, 'dsn')) 95 | 96 | def test_dsn(self): 97 | test_dsn = 'mysql://user:pass@host/db' 98 | self.assertEqual(test_dsn, value_types.ensure_type(test_dsn, 'dsn')) 99 | 100 | def test_dsn_set_variables(self): 101 | test_dsn = 'mysql://user:pass@host/db?charset=utf8' 102 | self.assertEqual(test_dsn, value_types.ensure_type(test_dsn, 'dsn')) 103 | 104 | def test_dsn_sqlite_memory(self): 105 | test_dsn = 'sqlite://' 106 | self.assertEqual(test_dsn, value_types.ensure_type(test_dsn, 'dsn')) 107 | 108 | def test_dsn_sqlite_file(self): 109 | test_dsn = 'sqlite:///tmp/foo.db' 110 | self.assertEqual(test_dsn, value_types.ensure_type(test_dsn, 'dsn')) 111 | 112 | def test_dsn_bad(self): 113 | self.assertRaises(config_exception.ConfigException, 114 | value_types.ensure_type, 115 | "mysql:/user:pass@host/db?charset=utf8", 'dsn') 116 | self.assertRaises(config_exception.ConfigException, 117 | value_types.ensure_type, 118 | "mysql://user:pass@host/db?charset=utf8;DROP TABLE " 119 | "foo", 'dsn') 120 | 121 | def test_swiftdevices_single(self): 122 | test_swiftdevices = 'r1z1-127.0.0.1:%PORT%/d1' 123 | self.assertEqual(test_swiftdevices, value_types.ensure_type( 124 | test_swiftdevices, 125 | 'swiftdevices')) 126 | 127 | def test_swiftdevices_multi(self): 128 | test_swiftdevices = 'r1z1-127.0.0.1:%PORT%/d1,r1z1-127.0.0.1:%PORT%/d2' 129 | self.assertEqual(test_swiftdevices, value_types.ensure_type( 130 | test_swiftdevices, 131 | 'swiftdevices')) 132 | 133 | def test_swiftdevices_blank(self): 134 | test_swiftdevices = '' 135 | self.assertRaises(config_exception.ConfigException, 136 | value_types.ensure_type, 137 | test_swiftdevices, 138 | 'swiftdevices') 139 | 140 | def test_swiftdevices_bad(self): 141 | test_swiftdevices = 'rz1-127.0.0.1:%PORT%/d1' 142 | self.assertRaises(config_exception.ConfigException, 143 | value_types.ensure_type, 144 | test_swiftdevices, 145 | 'swiftdevices') 146 | 147 | def test_username(self): 148 | for test_username in ['guest', 'guest_13-42']: 149 | self.assertEqual(test_username, value_types.ensure_type( 150 | test_username, 151 | 'username')) 152 | 153 | def test_username_bad(self): 154 | for test_username in ['guest`ls`', 'guest$PASSWD', 'guest 2']: 155 | self.assertRaises(config_exception.ConfigException, 156 | value_types.ensure_type, 157 | test_username, 158 | 'username') 159 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | -------------------------------------------------------------------------------- /os_apply_config/apply_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import argparse 17 | import json 18 | import logging 19 | import os 20 | import subprocess 21 | import sys 22 | import tempfile 23 | 24 | from pystache import context 25 | import yaml 26 | 27 | from os_apply_config import collect_config 28 | from os_apply_config import config_exception as exc 29 | from os_apply_config import oac_file 30 | from os_apply_config import renderers 31 | from os_apply_config import value_types 32 | from os_apply_config import version 33 | 34 | DEFAULT_TEMPLATES_DIR = '/usr/libexec/os-apply-config/templates' 35 | 36 | LOG_FORMAT = '[%(asctime)s] [%(levelname)s] %(message)s' 37 | DATE_FORMAT = '%Y/%m/%d %I:%M:%S %p' 38 | 39 | logger = logging.getLogger('os-apply-config') 40 | 41 | 42 | def templates_dir(): 43 | """Determine the default templates directory path 44 | 45 | If the OS_CONFIG_APPLIER_TEMPLATES environment variable has been set, 46 | use its value. 47 | Otherwise, select a default path based on which directories exist on the 48 | system, preferring the newer paths but still allowing the old ones for 49 | backwards compatibility. 50 | """ 51 | templates_dir = os.environ.get('OS_CONFIG_APPLIER_TEMPLATES', None) 52 | if templates_dir is None: 53 | templates_dir = '/opt/stack/os-apply-config/templates' 54 | if not os.path.isdir(templates_dir): 55 | # Backwards compat with the old name. 56 | templates_dir = '/opt/stack/os-config-applier/templates' 57 | if (os.path.isdir(templates_dir) and 58 | not os.path.isdir(DEFAULT_TEMPLATES_DIR)): 59 | logging.warning('Template directory %s is deprecated. The ' 60 | 'recommended location for template files is %s', 61 | templates_dir, DEFAULT_TEMPLATES_DIR) 62 | else: 63 | templates_dir = DEFAULT_TEMPLATES_DIR 64 | return templates_dir 65 | 66 | 67 | TEMPLATES_DIR = templates_dir() 68 | OS_CONFIG_FILES_PATH = os.environ.get( 69 | 'OS_CONFIG_FILES_PATH', '/var/lib/os-collect-config/os_config_files.json') 70 | OS_CONFIG_FILES_PATH_OLD = '/var/run/os-collect-config/os_config_files.json' 71 | 72 | CONTROL_FILE_SUFFIX = ".oac" 73 | 74 | 75 | def install_config( 76 | config_path, template_root, output_path, validate, subhash=None, 77 | fallback_metadata=None): 78 | config = strip_hash( 79 | collect_config.collect_config(config_path, fallback_metadata), subhash) 80 | tree = build_tree(template_paths(template_root), config) 81 | if not validate: 82 | for path, obj in tree.items(): 83 | write_file(os.path.join( 84 | output_path, strip_prefix('/', path)), obj) 85 | 86 | 87 | def _extract_key(config_path, key, fallback_metadata=None): 88 | config = collect_config.collect_config(config_path, fallback_metadata) 89 | keys = key.split('.') 90 | for key in keys: 91 | try: 92 | config = config[key] 93 | if config is None: 94 | raise TypeError() 95 | except (KeyError, TypeError): 96 | try: 97 | if isinstance(config, list): 98 | config = config[int(key)] 99 | continue 100 | except (IndexError, ValueError): 101 | pass 102 | return None 103 | return config 104 | 105 | 106 | def print_key( 107 | config_path, key, type_name, default=None, fallback_metadata=None): 108 | config = collect_config.collect_config(config_path, fallback_metadata) 109 | config = _extract_key(config_path, key, fallback_metadata) 110 | if config is None: 111 | if default is not None: 112 | print(str(default)) 113 | return 114 | else: 115 | raise exc.ConfigException( 116 | 'key {} does not exist in {}'.format(key, config_path)) 117 | value_types.ensure_type(str(config), type_name) 118 | if isinstance(config, (dict, list, bool)): 119 | print(json.dumps(config)) 120 | else: 121 | print(str(config)) 122 | 123 | 124 | def boolean_key(metadata, key, fallback_metadata): 125 | config = _extract_key(metadata, key, fallback_metadata) 126 | if not isinstance(config, bool): 127 | return -1 128 | if config: 129 | return 0 130 | else: 131 | return 1 132 | 133 | 134 | def write_file(path, obj): 135 | if not obj.allow_empty and len(obj.body) == 0: 136 | if os.path.exists(path): 137 | logger.info("deleting %s", path) 138 | os.unlink(path) 139 | else: 140 | logger.info("not creating empty %s", path) 141 | return 142 | 143 | logger.info("writing %s", path) 144 | if os.path.exists(path): 145 | stat = os.stat(path) 146 | mode, uid, gid = stat.st_mode, stat.st_uid, stat.st_gid 147 | else: 148 | mode, uid, gid = 0o644, -1, -1 149 | mode = obj.mode or mode 150 | if obj.owner is not None: 151 | uid = obj.owner 152 | if obj.group is not None: 153 | gid = obj.group 154 | 155 | d = os.path.dirname(path) 156 | os.path.exists(d) or os.makedirs(d) 157 | with tempfile.NamedTemporaryFile(dir=d, delete=False) as newfile: 158 | if isinstance(obj.body, str): 159 | obj.body = obj.body.encode('utf-8') 160 | newfile.write(obj.body) 161 | os.chmod(newfile.name, mode) 162 | os.chown(newfile.name, uid, gid) 163 | os.rename(newfile.name, path) 164 | 165 | 166 | def build_tree(templates, config): 167 | """Return a map of filenames to OacFiles.""" 168 | res = {} 169 | for in_file, out_file in templates: 170 | try: 171 | body = render_template(in_file, config) 172 | ctrl_file = in_file + CONTROL_FILE_SUFFIX 173 | ctrl_dict = {} 174 | if os.path.isfile(ctrl_file): 175 | with open(ctrl_file) as cf: 176 | ctrl_body = cf.read() 177 | ctrl_dict = yaml.safe_load(ctrl_body) or {} 178 | if not isinstance(ctrl_dict, dict): 179 | raise exc.ConfigException( 180 | "header is not a dict: %s" % in_file) 181 | res[out_file] = oac_file.OacFile(body, **ctrl_dict) 182 | except exc.ConfigException as e: 183 | e.args += in_file, 184 | raise 185 | return res 186 | 187 | 188 | def render_template(template, config): 189 | if is_executable(template): 190 | return render_executable(template, config) 191 | else: 192 | try: 193 | return render_moustache(open(template).read(), config) 194 | except context.KeyNotFoundError as e: 195 | raise exc.ConfigException( 196 | "key '%s' from template '%s' does not exist in metadata file." 197 | % (e.key, template)) 198 | except Exception as e: 199 | logger.error("%s", e) 200 | raise exc.ConfigException( 201 | "could not render moustache template %s" % template) 202 | 203 | 204 | def is_executable(path): 205 | return os.path.isfile(path) and os.access(path, os.X_OK) 206 | 207 | 208 | def render_moustache(text, config): 209 | r = renderers.JsonRenderer(missing_tags='ignore') 210 | return r.render(text, config) 211 | 212 | 213 | def render_executable(path, config): 214 | p = subprocess.Popen([path], 215 | stdin=subprocess.PIPE, 216 | stdout=subprocess.PIPE, 217 | stderr=subprocess.PIPE) 218 | stdout, stderr = p.communicate(json.dumps(config).encode('utf-8')) 219 | p.wait() 220 | if p.returncode != 0: 221 | raise exc.ConfigException( 222 | "config script failed: %s\n\nwith output:\n\n%s" % 223 | (path, stdout + stderr)) 224 | return stdout.decode('utf-8') 225 | 226 | 227 | def template_paths(root): 228 | res = [] 229 | for cur_root, _subdirs, files in os.walk(root): 230 | for f in files: 231 | if f.endswith(CONTROL_FILE_SUFFIX): 232 | continue 233 | inout = (os.path.join(cur_root, f), os.path.join( 234 | strip_prefix(root, cur_root), f)) 235 | res.append(inout) 236 | return res 237 | 238 | 239 | def strip_prefix(prefix, s): 240 | return s[len(prefix):] if s.startswith(prefix) else s 241 | 242 | 243 | def strip_hash(h, keys): 244 | if not keys: 245 | return h 246 | for k in keys.split('.'): 247 | if k in h and isinstance(h[k], dict): 248 | h = h[k] 249 | else: 250 | raise exc.ConfigException( 251 | "key '%s' does not correspond to a hash in the metadata file" 252 | % keys) 253 | return h 254 | 255 | 256 | def parse_opts(argv): 257 | parser = argparse.ArgumentParser( 258 | description='Reads and merges JSON configuration files specified' 259 | ' by colon separated environment variable OS_CONFIG_FILES, unless' 260 | ' overridden by command line option --metadata. If no files are' 261 | ' specified this way, falls back to legacy behavior of searching' 262 | ' the fallback metadata path for a single config file.') 263 | parser.add_argument('-t', '--templates', metavar='TEMPLATE_ROOT', 264 | help="""path to template root directory (default: 265 | %(default)s)""", 266 | default=TEMPLATES_DIR) 267 | parser.add_argument('-o', '--output', metavar='OUT_DIR', 268 | help='root directory for output (default:%(default)s)', 269 | default='/') 270 | parser.add_argument('-m', '--metadata', metavar='METADATA_FILE', nargs='*', 271 | help='Overrides environment variable OS_CONFIG_FILES.' 272 | ' Specify multiple times, rather than separate files' 273 | ' with ":".', 274 | default=[]) 275 | parser.add_argument('--fallback-metadata', metavar='FALLBACK_METADATA', 276 | nargs='*', help='Files to search when OS_CONFIG_FILES' 277 | ' is empty. (default: %(default)s)', 278 | default=['/var/cache/heat-cfntools/last_metadata', 279 | '/var/lib/heat-cfntools/cfn-init-data', 280 | '/var/lib/cloud/data/cfn-init-data']) 281 | parser.add_argument( 282 | '-v', '--validate', help='validate only. do not write files', 283 | default=False, action='store_true') 284 | parser.add_argument( 285 | '--print-templates', default=False, action='store_true', 286 | help='Print templates root and exit.') 287 | parser.add_argument('-s', '--subhash', 288 | help='use the sub-hash named by this key,' 289 | ' instead of the full metadata hash') 290 | parser.add_argument('--key', metavar='KEY', default=None, 291 | help='print the specified key and exit.' 292 | ' (may be used with --type and --key-default)') 293 | parser.add_argument('--type', default='default', 294 | help='exit with error if the specified --key does not' 295 | ' match type. Valid types are' 296 | ' ') 298 | parser.add_argument('--key-default', 299 | help='This option only affects running with --key.' 300 | ' Print this if key is not found. This value is' 301 | ' not subject to type restrictions. If --key is' 302 | ' specified and no default is specified, program' 303 | ' exits with an error on missing key.') 304 | parser.add_argument('--boolean-key', 305 | help='This option is incompatible with --key.' 306 | ' Use this to evaluate whether a value is' 307 | ' boolean true or false. The return code of the' 308 | ' command will be 0 for true, 1 for false, and -1' 309 | ' for non-boolean values.') 310 | parser.add_argument('--version', action='version', 311 | version=version.version_info.version_string()) 312 | parser.add_argument('--os-config-files', 313 | default=OS_CONFIG_FILES_PATH, 314 | help='Set path to os_config_files.json') 315 | opts = parser.parse_args(argv[1:]) 316 | 317 | return opts 318 | 319 | 320 | def load_list_from_json(json_file): 321 | json_obj = [] 322 | if os.path.exists(json_file): 323 | with open(json_file) as ocf: 324 | json_obj = json.loads(ocf.read()) 325 | if not isinstance(json_obj, list): 326 | raise ValueError("No list defined in json file: %s" % json_file) 327 | return json_obj 328 | 329 | 330 | def add_handler(logger, handler): 331 | handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT)) 332 | logger.addHandler(handler) 333 | 334 | 335 | def main(argv=sys.argv): 336 | opts = parse_opts(argv) 337 | if opts.print_templates: 338 | print(opts.templates) 339 | return 0 340 | 341 | logger.setLevel(logging.INFO) 342 | add_handler(logger, logging.StreamHandler()) 343 | if os.geteuid() == 0: 344 | add_handler(logger, 345 | logging.FileHandler('/var/log/os-apply-config.log')) 346 | 347 | if not opts.metadata: 348 | if 'OS_CONFIG_FILES' in os.environ: 349 | opts.metadata = os.environ['OS_CONFIG_FILES'].split(':') 350 | else: 351 | opts.metadata = load_list_from_json(opts.os_config_files) 352 | if (not opts.metadata and opts.os_config_files == 353 | OS_CONFIG_FILES_PATH): 354 | logger.warning('DEPRECATED: falling back to %s' % 355 | OS_CONFIG_FILES_PATH_OLD) 356 | opts.metadata = load_list_from_json(OS_CONFIG_FILES_PATH_OLD) 357 | 358 | if opts.key and opts.boolean_key: 359 | logger.warning('--key is not compatible with --boolean-key.' 360 | ' --boolean-key ignored.') 361 | 362 | try: 363 | if opts.templates is None: 364 | raise exc.ConfigException('missing option --templates') 365 | 366 | if opts.key: 367 | print_key(opts.metadata, 368 | opts.key, 369 | opts.type, 370 | opts.key_default, 371 | opts.fallback_metadata) 372 | elif opts.boolean_key: 373 | return boolean_key(opts.metadata, 374 | opts.boolean_key, 375 | opts.fallback_metadata) 376 | else: 377 | install_config(opts.metadata, opts.templates, opts.output, 378 | opts.validate, opts.subhash, opts.fallback_metadata) 379 | logger.info("success") 380 | except exc.ConfigException as e: 381 | logger.error(e) 382 | return 1 383 | return 0 384 | -------------------------------------------------------------------------------- /os_apply_config/tests/test_apply_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import atexit 17 | import json 18 | import os 19 | import tempfile 20 | from unittest import mock 21 | 22 | import fixtures 23 | import testtools 24 | 25 | from os_apply_config import apply_config 26 | from os_apply_config import config_exception as exc 27 | from os_apply_config import oac_file 28 | 29 | # example template tree 30 | TEMPLATES = os.path.join(os.path.dirname(__file__), 'templates') 31 | 32 | # config for example tree 33 | CONFIG = { 34 | "x": "foo", 35 | "y": False, 36 | "z": None, 37 | "btrue": True, 38 | "bfalse": False, 39 | "database": { 40 | "url": "sqlite:///blah" 41 | }, 42 | "l": [1, 2], 43 | } 44 | 45 | # config for example tree - with subhash 46 | CONFIG_SUBHASH = { 47 | "OpenStack::Config": { 48 | "x": "foo", 49 | "database": { 50 | "url": "sqlite:///blah" 51 | } 52 | } 53 | } 54 | 55 | # expected output for example tree 56 | OUTPUT = { 57 | "/etc/glance/script.conf": oac_file.OacFile( 58 | "foo\n"), 59 | "/etc/keystone/keystone.conf": oac_file.OacFile( 60 | "[foo]\ndatabase = sqlite:///blah\n"), 61 | "/etc/control/empty": oac_file.OacFile( 62 | "foo\n"), 63 | "/etc/control/allow_empty": oac_file.OacFile( 64 | "").set('allow_empty', False), 65 | "/etc/control/mode": oac_file.OacFile( 66 | "lorem modus\n").set('mode', 0o755), 67 | } 68 | TEMPLATE_PATHS = OUTPUT.keys() 69 | 70 | # expected output for chown tests 71 | # separated out to avoid needing to mock os.chown for most tests 72 | CHOWN_TEMPLATES = os.path.join(os.path.dirname(__file__), 'chown_templates') 73 | CHOWN_OUTPUT = { 74 | "owner.uid": oac_file.OacFile("lorem uido\n").set('owner', 0), 75 | "owner.name": oac_file.OacFile("namo uido\n").set('owner', 0), 76 | "group.gid": oac_file.OacFile("lorem gido\n").set('group', 0), 77 | "group.name": oac_file.OacFile("namo gido\n").set('group', 0), 78 | } 79 | 80 | 81 | def template(relpath): 82 | return os.path.join(TEMPLATES, relpath[1:]) 83 | 84 | 85 | class TestRunOSConfigApplier(testtools.TestCase): 86 | """Tests the commandline options.""" 87 | 88 | def setUp(self): 89 | super().setUp() 90 | self.useFixture(fixtures.NestedTempfile()) 91 | self.stdout = self.useFixture(fixtures.StringStream('stdout')).stream 92 | self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.stdout)) 93 | stderr = self.useFixture(fixtures.StringStream('stderr')).stream 94 | self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) 95 | self.logger = self.useFixture( 96 | fixtures.FakeLogger(name="os-apply-config")) 97 | fd, self.path = tempfile.mkstemp() 98 | with os.fdopen(fd, 'w') as t: 99 | t.write(json.dumps(CONFIG)) 100 | t.flush() 101 | 102 | def test_print_key(self): 103 | self.assertEqual(0, apply_config.main( 104 | ['os-apply-config.py', '--metadata', self.path, '--key', 105 | 'database.url', '--type', 'raw'])) 106 | self.stdout.seek(0) 107 | self.assertEqual(CONFIG['database']['url'], 108 | self.stdout.read().strip()) 109 | self.assertEqual('', self.logger.output) 110 | 111 | def test_print_key_json_dict(self): 112 | self.assertEqual(0, apply_config.main( 113 | ['os-apply-config.py', '--metadata', self.path, '--key', 114 | 'database', '--type', 'raw'])) 115 | self.stdout.seek(0) 116 | self.assertEqual(CONFIG['database'], 117 | json.loads(self.stdout.read().strip())) 118 | self.assertEqual('', self.logger.output) 119 | 120 | def test_print_key_json_list(self): 121 | self.assertEqual(0, apply_config.main( 122 | ['os-apply-config.py', '--metadata', self.path, '--key', 123 | 'l', '--type', 'raw'])) 124 | self.stdout.seek(0) 125 | self.assertEqual(CONFIG['l'], 126 | json.loads(self.stdout.read().strip())) 127 | self.assertEqual('', self.logger.output) 128 | 129 | def test_print_non_string_key(self): 130 | self.assertEqual(0, apply_config.main( 131 | ['os-apply-config.py', '--metadata', self.path, '--key', 132 | 'y', '--type', 'raw'])) 133 | self.stdout.seek(0) 134 | self.assertEqual("false", 135 | self.stdout.read().strip()) 136 | self.assertEqual('', self.logger.output) 137 | 138 | def test_print_null_key(self): 139 | self.assertEqual(0, apply_config.main( 140 | ['os-apply-config.py', '--metadata', self.path, '--key', 141 | 'z', '--type', 'raw', '--key-default', ''])) 142 | self.stdout.seek(0) 143 | self.assertEqual('', self.stdout.read().strip()) 144 | self.assertEqual('', self.logger.output) 145 | 146 | def test_print_key_missing(self): 147 | self.assertEqual(1, apply_config.main( 148 | ['os-apply-config.py', '--metadata', self.path, '--key', 149 | 'does.not.exist'])) 150 | self.assertIn('does not exist', self.logger.output) 151 | 152 | def test_print_key_missing_default(self): 153 | self.assertEqual(0, apply_config.main( 154 | ['os-apply-config.py', '--metadata', self.path, '--key', 155 | 'does.not.exist', '--key-default', ''])) 156 | self.stdout.seek(0) 157 | self.assertEqual('', self.stdout.read().strip()) 158 | self.assertEqual('', self.logger.output) 159 | 160 | def test_print_key_wrong_type(self): 161 | self.assertEqual(1, apply_config.main( 162 | ['os-apply-config.py', '--metadata', self.path, '--key', 163 | 'x', '--type', 'int'])) 164 | self.assertIn('cannot interpret value', self.logger.output) 165 | 166 | def test_print_key_from_list(self): 167 | self.assertEqual(0, apply_config.main( 168 | ['os-apply-config.py', '--metadata', self.path, '--key', 169 | 'l.0', '--type', 'int'])) 170 | self.stdout.seek(0) 171 | self.assertEqual(str(CONFIG['l'][0]), 172 | self.stdout.read().strip()) 173 | self.assertEqual('', self.logger.output) 174 | 175 | def test_print_key_from_list_missing(self): 176 | self.assertEqual(1, apply_config.main( 177 | ['os-apply-config.py', '--metadata', self.path, '--key', 178 | 'l.2', '--type', 'int'])) 179 | self.assertIn('does not exist', self.logger.output) 180 | 181 | def test_print_key_from_list_missing_default(self): 182 | self.assertEqual(0, apply_config.main( 183 | ['os-apply-config.py', '--metadata', self.path, '--key', 184 | 'l.2', '--type', 'int', '--key-default', ''])) 185 | self.stdout.seek(0) 186 | self.assertEqual('', self.stdout.read().strip()) 187 | self.assertEqual('', self.logger.output) 188 | 189 | def test_print_templates(self): 190 | apply_config.main(['os-apply-config', '--print-templates']) 191 | self.stdout.seek(0) 192 | self.assertEqual( 193 | self.stdout.read().strip(), apply_config.TEMPLATES_DIR) 194 | self.assertEqual('', self.logger.output) 195 | 196 | def test_boolean_key(self): 197 | rcode = apply_config.main(['os-apply-config', '--metadata', 198 | self.path, '--boolean-key', 'btrue']) 199 | self.assertEqual(0, rcode) 200 | rcode = apply_config.main(['os-apply-config', '--metadata', 201 | self.path, '--boolean-key', 'bfalse']) 202 | self.assertEqual(1, rcode) 203 | rcode = apply_config.main(['os-apply-config', '--metadata', 204 | self.path, '--boolean-key', 'x']) 205 | self.assertEqual(-1, rcode) 206 | 207 | def test_boolean_key_and_key(self): 208 | rcode = apply_config.main(['os-apply-config', '--metadata', 209 | self.path, '--boolean-key', 'btrue', 210 | '--key', 'x']) 211 | self.assertEqual(0, rcode) 212 | self.stdout.seek(0) 213 | self.assertEqual(self.stdout.read().strip(), 'foo') 214 | self.assertIn('--boolean-key ignored', self.logger.output) 215 | 216 | def test_os_config_files(self): 217 | with tempfile.NamedTemporaryFile() as fake_os_config_files: 218 | with tempfile.NamedTemporaryFile() as fake_config: 219 | fake_config.write(json.dumps(CONFIG).encode('utf-8')) 220 | fake_config.flush() 221 | fake_os_config_files.write( 222 | json.dumps([fake_config.name]).encode('utf-8')) 223 | fake_os_config_files.flush() 224 | apply_config.main(['os-apply-config', 225 | '--key', 'database.url', 226 | '--type', 'raw', 227 | '--os-config-files', 228 | fake_os_config_files.name]) 229 | self.stdout.seek(0) 230 | self.assertEqual( 231 | CONFIG['database']['url'], self.stdout.read().strip()) 232 | 233 | 234 | class OSConfigApplierTestCase(testtools.TestCase): 235 | 236 | def setUp(self): 237 | super().setUp() 238 | self.logger = self.useFixture(fixtures.FakeLogger('os-apply-config')) 239 | self.useFixture(fixtures.NestedTempfile()) 240 | 241 | def write_config(self, config): 242 | fd, path = tempfile.mkstemp() 243 | with os.fdopen(fd, 'w') as t: 244 | t.write(json.dumps(config)) 245 | t.flush() 246 | return path 247 | 248 | def check_output_file(self, tmpdir, path, obj): 249 | full_path = os.path.join(tmpdir, path[1:]) 250 | if obj.allow_empty: 251 | assert os.path.exists(full_path), "%s doesn't exist" % path 252 | self.assertEqual(obj.body, open(full_path).read()) 253 | else: 254 | assert not os.path.exists(full_path), "%s exists" % path 255 | 256 | def test_install_config(self): 257 | path = self.write_config(CONFIG) 258 | tmpdir = tempfile.mkdtemp() 259 | apply_config.install_config([path], TEMPLATES, tmpdir, False) 260 | for path, obj in OUTPUT.items(): 261 | self.check_output_file(tmpdir, path, obj) 262 | 263 | def test_install_config_subhash(self): 264 | tpath = self.write_config(CONFIG_SUBHASH) 265 | tmpdir = tempfile.mkdtemp() 266 | apply_config.install_config( 267 | [tpath], TEMPLATES, tmpdir, False, 'OpenStack::Config') 268 | for path, obj in OUTPUT.items(): 269 | self.check_output_file(tmpdir, path, obj) 270 | 271 | def test_delete_if_not_allowed_empty(self): 272 | path = self.write_config(CONFIG) 273 | tmpdir = tempfile.mkdtemp() 274 | template = "/etc/control/allow_empty" 275 | target_file = os.path.join(tmpdir, template[1:]) 276 | # Touch the file 277 | os.makedirs(os.path.dirname(target_file)) 278 | open(target_file, 'a').close() 279 | apply_config.install_config([path], TEMPLATES, tmpdir, False) 280 | # File should be gone 281 | self.assertFalse(os.path.exists(target_file)) 282 | 283 | def test_respect_file_permissions(self): 284 | path = self.write_config(CONFIG) 285 | tmpdir = tempfile.mkdtemp() 286 | template = "/etc/keystone/keystone.conf" 287 | target_file = os.path.join(tmpdir, template[1:]) 288 | os.makedirs(os.path.dirname(target_file)) 289 | # File doesn't exist, use the default mode (644) 290 | apply_config.install_config([path], TEMPLATES, tmpdir, False) 291 | self.assertEqual(0o100644, os.stat(target_file).st_mode) 292 | self.assertEqual(OUTPUT[template].body, open(target_file).read()) 293 | # Set a different mode: 294 | os.chmod(target_file, 0o600) 295 | apply_config.install_config([path], TEMPLATES, tmpdir, False) 296 | # The permissions should be preserved 297 | self.assertEqual(0o100600, os.stat(target_file).st_mode) 298 | self.assertEqual(OUTPUT[template].body, open(target_file).read()) 299 | 300 | def test_build_tree(self): 301 | tree = apply_config.build_tree( 302 | apply_config.template_paths(TEMPLATES), CONFIG) 303 | self.assertEqual(OUTPUT, tree) 304 | 305 | def test_render_template(self): 306 | # execute executable files, moustache non-executables 307 | self.assertEqual("abc\n", apply_config.render_template(template( 308 | "/etc/glance/script.conf"), {"x": "abc"})) 309 | self.assertRaises( 310 | exc.ConfigException, 311 | apply_config.render_template, 312 | template("/etc/glance/script.conf"), {}) 313 | 314 | def test_render_template_bad_template(self): 315 | tdir = self.useFixture(fixtures.TempDir()) 316 | bt_path = os.path.join(tdir.path, 'bad_template') 317 | with open(bt_path, 'w') as bt: 318 | bt.write("{{#foo}}bar={{bar}}{{/bar}}") 319 | e = self.assertRaises(exc.ConfigException, 320 | apply_config.render_template, 321 | bt_path, {'foo': [{'bar': 322 | 'abc'}]}) 323 | self.assertIn('could not render moustache template', str(e)) 324 | self.assertIn('Section end tag mismatch', self.logger.output) 325 | 326 | def test_render_moustache(self): 327 | self.assertEqual( 328 | "ab123cd", 329 | apply_config.render_moustache("ab{{x.a}}cd", {"x": {"a": "123"}})) 330 | 331 | def test_render_moustache_bad_key(self): 332 | self.assertEqual('', apply_config.render_moustache("{{badkey}}", {})) 333 | 334 | def test_render_moustache_none(self): 335 | self.assertEqual('foo: ', 336 | apply_config.render_moustache("foo: {{foo}}", 337 | {'foo': None})) 338 | 339 | def test_render_executable(self): 340 | params = {"x": "foo"} 341 | self.assertEqual("foo\n", apply_config.render_executable( 342 | template("/etc/glance/script.conf"), params)) 343 | 344 | def test_render_executable_failure(self): 345 | self.assertRaises( 346 | exc.ConfigException, 347 | apply_config.render_executable, 348 | template("/etc/glance/script.conf"), {}) 349 | 350 | def test_template_paths(self): 351 | expected = list(map(lambda p: (template(p), p), TEMPLATE_PATHS)) 352 | actual = apply_config.template_paths(TEMPLATES) 353 | expected.sort(key=lambda tup: tup[1]) 354 | actual.sort(key=lambda tup: tup[1]) 355 | self.assertEqual(expected, actual) 356 | 357 | def test_strip_hash(self): 358 | h = {'a': {'b': {'x': 'y'}}, "c": [1, 2, 3]} 359 | self.assertEqual({'x': 'y'}, apply_config.strip_hash(h, 'a.b')) 360 | self.assertRaises(exc.ConfigException, 361 | apply_config.strip_hash, h, 'a.nonexistent') 362 | self.assertRaises(exc.ConfigException, 363 | apply_config.strip_hash, h, 'a.c') 364 | 365 | def test_load_list_from_json(self): 366 | def mkstemp(): 367 | fd, path = tempfile.mkstemp() 368 | atexit.register( 369 | lambda: os.path.exists(path) and os.remove(path)) 370 | return (fd, path) 371 | 372 | def write_contents(fd, contents): 373 | with os.fdopen(fd, 'w') as t: 374 | t.write(contents) 375 | t.flush() 376 | 377 | fd, path = mkstemp() 378 | load_list = apply_config.load_list_from_json 379 | self.assertRaises(ValueError, load_list, path) 380 | write_contents(fd, json.dumps(["/tmp/config.json"])) 381 | json_obj = load_list(path) 382 | self.assertEqual(["/tmp/config.json"], json_obj) 383 | os.remove(path) 384 | self.assertEqual([], load_list(path)) 385 | 386 | fd, path = mkstemp() 387 | write_contents(fd, json.dumps({})) 388 | self.assertRaises(ValueError, load_list, path) 389 | 390 | def test_default_templates_dir_current(self): 391 | default = '/usr/libexec/os-apply-config/templates' 392 | with mock.patch('os.path.isdir', lambda x: x == default): 393 | self.assertEqual(default, apply_config.templates_dir()) 394 | 395 | def test_default_templates_dir_deprecated(self): 396 | default = '/opt/stack/os-apply-config/templates' 397 | with mock.patch('os.path.isdir', lambda x: x == default): 398 | self.assertEqual(default, apply_config.templates_dir()) 399 | 400 | def test_default_templates_dir_old_deprecated(self): 401 | default = '/opt/stack/os-config-applier/templates' 402 | with mock.patch('os.path.isdir', lambda x: x == default): 403 | self.assertEqual(default, apply_config.templates_dir()) 404 | 405 | def test_default_templates_dir_both(self): 406 | default = '/usr/libexec/os-apply-config/templates' 407 | deprecated = '/opt/stack/os-apply-config/templates' 408 | with mock.patch('os.path.isdir', lambda x: (x == default or 409 | x == deprecated)): 410 | self.assertEqual(default, apply_config.templates_dir()) 411 | 412 | def test_control_mode(self): 413 | path = self.write_config(CONFIG) 414 | tmpdir = tempfile.mkdtemp() 415 | template = "/etc/control/mode" 416 | target_file = os.path.join(tmpdir, template[1:]) 417 | apply_config.install_config([path], TEMPLATES, tmpdir, False) 418 | self.assertEqual(0o100755, os.stat(target_file).st_mode) 419 | 420 | @mock.patch('os.chown') 421 | def test_control_chown(self, chown_mock): 422 | path = self.write_config(CONFIG) 423 | tmpdir = tempfile.mkdtemp() 424 | apply_config.install_config([path], CHOWN_TEMPLATES, tmpdir, False) 425 | chown_mock.assert_has_calls([mock.call(mock.ANY, 0, -1), # uid 426 | mock.call(mock.ANY, 0, -1), # username 427 | mock.call(mock.ANY, -1, 0), # gid 428 | mock.call(mock.ANY, -1, 0)], # groupname 429 | any_order=True) 430 | --------------------------------------------------------------------------------