├── examples ├── host │ ├── country │ │ └── tests │ │ │ └── country-1.yaml │ ├── region │ │ ├── tests │ │ │ └── region-1.yaml │ │ ├── apac.yaml │ │ ├── amer.yaml │ │ └── emea.yaml │ ├── roles │ │ └── tests │ │ │ └── roles-1.yaml │ ├── environment │ │ └── tests │ │ │ ├── country-1.yaml │ │ │ └── environment-1.yaml │ ├── override │ │ ├── tests │ │ │ └── override-1.yaml │ │ └── test_example_com.yaml │ ├── datacenter │ │ └── tests │ │ │ └── datacenter-1.yaml │ ├── input │ │ ├── tests │ │ │ └── input-1.yaml │ │ └── test_example_com.yaml │ ├── default │ │ ├── tests │ │ │ └── default-1.yaml │ │ └── default.yaml │ ├── osfinger │ │ ├── tests │ │ │ └── osfinger-1.yaml │ │ └── fedora_19.yaml │ └── schemas │ │ ├── mail.yaml │ │ ├── location.yaml │ │ ├── time.yaml │ │ ├── salt.yaml │ │ ├── host.yaml │ │ ├── pkgrepo.yaml │ │ └── network.yaml └── master ├── requirements.txt ├── Makefile ├── .gitignore ├── tests ├── test_all.sh ├── test_pylint.sh ├── test_output.py └── output.yaml ├── .pylintrc ├── LICENSE ├── setup.py ├── scripts ├── pepa-test └── pepa ├── README.rst ├── pillar └── pepa.py └── pepa └── __init__.py /examples/host/country/tests/country-1.yaml: -------------------------------------------------------------------------------- 1 | test: dummy 2 | -------------------------------------------------------------------------------- /examples/host/region/tests/region-1.yaml: -------------------------------------------------------------------------------- 1 | test: dummy 2 | -------------------------------------------------------------------------------- /examples/host/roles/tests/roles-1.yaml: -------------------------------------------------------------------------------- 1 | test: dummy 2 | -------------------------------------------------------------------------------- /examples/host/environment/tests/country-1.yaml: -------------------------------------------------------------------------------- 1 | test: dummy 2 | -------------------------------------------------------------------------------- /examples/host/override/tests/override-1.yaml: -------------------------------------------------------------------------------- 1 | test: dummy 2 | -------------------------------------------------------------------------------- /examples/host/datacenter/tests/datacenter-1.yaml: -------------------------------------------------------------------------------- 1 | test: dummy 2 | -------------------------------------------------------------------------------- /examples/host/environment/tests/environment-1.yaml: -------------------------------------------------------------------------------- 1 | test: dummy 2 | -------------------------------------------------------------------------------- /examples/host/input/tests/input-1.yaml: -------------------------------------------------------------------------------- 1 | hostname: test.example.com 2 | -------------------------------------------------------------------------------- /examples/host/override/test_example_com.yaml: -------------------------------------------------------------------------------- 1 | network..dns..servers..merge(): 2 | - 10.2.0.3 3 | -------------------------------------------------------------------------------- /examples/host/default/tests/default-1.yaml: -------------------------------------------------------------------------------- 1 | location..region: emea 2 | grains..osfinger: Fedora-20 3 | -------------------------------------------------------------------------------- /examples/host/osfinger/tests/osfinger-1.yaml: -------------------------------------------------------------------------------- 1 | osfinger: Fedora 20 2 | pkgrepo..mirror: yum.emea.com 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml 2 | jinja2 3 | argparse 4 | logging 5 | colorlog 6 | cerberus 7 | requests 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: publish 2 | 3 | clean: 4 | rm -rf dist pepa.egg-info 5 | 6 | publish: clean 7 | python setup.py sdist 8 | twine upload dist/* 9 | 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *.swp 4 | 5 | # Ignore virtualenv 6 | bin/ 7 | include/ 8 | lib/ 9 | lib64/ 10 | 11 | # PyPi package 12 | dist/ 13 | pepa.egg-info/ 14 | -------------------------------------------------------------------------------- /tests/test_all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -u 5 | 6 | for test in $( ls tests/*.{sh,py} | grep -v test_all.sh ); do 7 | printf "\n#### ${test} ####\n\n" 8 | $test 9 | done 10 | -------------------------------------------------------------------------------- /examples/host/schemas/mail.yaml: -------------------------------------------------------------------------------- 1 | mail..agent: 2 | type: string 3 | allowed: [ 'sendmail', 'postfix' ] 4 | 5 | mail..gateway: 6 | type: string 7 | regex: ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-zA-Z]{2,6}$ 8 | -------------------------------------------------------------------------------- /examples/host/schemas/location.yaml: -------------------------------------------------------------------------------- 1 | location..region: 2 | type: string 3 | allowed: [ 'amer', 'apac', 'emea' ] 4 | 5 | location..country: 6 | type: string 7 | regex: ^[a-z]{2}$ 8 | 9 | location..datacenter: 10 | type: string 11 | regex: ^[a-z0-9]+$ 12 | -------------------------------------------------------------------------------- /tests/test_pylint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -u 5 | 6 | exit_code=0 7 | for file in pepa/__init__.py scripts/pepa scripts/pepa-test; do 8 | echo $file 9 | pylint --rcfile .pylintrc $file || exit_code=1 10 | done 11 | exit $exit_code 12 | -------------------------------------------------------------------------------- /examples/host/region/apac.yaml: -------------------------------------------------------------------------------- 1 | time..ntp..servers: 2 | - ntp1.apac.example.com 3 | - ntp2.apac.example.com 4 | - ntp3.apac.example.com 5 | network..dns..servers: 6 | - 10.1.0.1 7 | - 10.1.0.2 8 | time..timezone: Asia/Hong_Kong 9 | pkgrepo..mirror: yum.apac.example.com 10 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | disable=no-member,maybe-no-member,line-too-long,invalid-name,import-error,too-many-branches,too-many-locals,too-many-statements,too-many-arguments,dangerous-default-value,too-few-public-methods,broad-except,import-self 3 | [REPORTS] 4 | reports=no 5 | -------------------------------------------------------------------------------- /examples/host/region/amer.yaml: -------------------------------------------------------------------------------- 1 | network..dns..servers: 2 | - 10.0.0.1 3 | - 10.0.0.2 4 | time..ntp..servers: 5 | - ntp1.amer.example.com 6 | - ntp2.amer.example.com 7 | - ntp3.amer.example.com 8 | time..timezone: America/Chihuahua 9 | pkgrepo..mirror: yum.amer.example.com 10 | -------------------------------------------------------------------------------- /examples/host/region/emea.yaml: -------------------------------------------------------------------------------- 1 | network..dns..servers: 2 | - 10.2.0.1 3 | - 10.2.0.2 4 | time..ntp..servers: 5 | - ntp1.amer.example.com 6 | - ntp2.amer.example.com 7 | - ntp3.amer.example.com 8 | time..timezone: Europe/Stockholm 9 | pkgrepo..mirror: yum.emea.example.com 10 | -------------------------------------------------------------------------------- /examples/host/schemas/time.yaml: -------------------------------------------------------------------------------- 1 | time..ptp..enabled: 2 | type: boolean 3 | 4 | time..timezone: 5 | type: string 6 | regex: ^([A-Za-z_]+/*)+$ 7 | 8 | time..ntp..servers: 9 | type: list 10 | schema: 11 | regex: ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-zA-Z]{2,6}$ 12 | -------------------------------------------------------------------------------- /examples/host/schemas/salt.yaml: -------------------------------------------------------------------------------- 1 | {% set version = '^[0-9\.]+$' %} 2 | 3 | salt..master: 4 | type: list 5 | schema: 6 | regex: ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-zA-Z]{2,6}$ 7 | 8 | salt..version: 9 | type: string 10 | regex: {{ version }} 11 | 12 | salt..release: 13 | type: string 14 | regex: {{ version }} 15 | -------------------------------------------------------------------------------- /examples/host/default/default.yaml: -------------------------------------------------------------------------------- 1 | network..dns..search: 2 | - example.com 3 | network..dns..options: 4 | - timeout:2 5 | - attempts:1 6 | - ndots:1 7 | mail..agent: sendmail 8 | mail..gateway: smtp.{{ location['region'] }}.example.com 9 | osfinger: {{ grains.osfinger }} 10 | salt..version: '2014.1.5' 11 | salt..release: '1' 12 | salt..master: 13 | - salt.{{ location['region'] }}.example.com 14 | -------------------------------------------------------------------------------- /examples/host/schemas/host.yaml: -------------------------------------------------------------------------------- 1 | osfinger: 2 | type: string 3 | regex: ^[A-Za-z0-9\ \-\.]+$ 4 | 5 | hostname: 6 | type: string 7 | regex: {{ hostname }} 8 | 9 | environment: 10 | type: string 11 | allowed: [ 'base', 'dev', 'qa', 'pilot', 'prod' ] 12 | 13 | roles: 14 | type: list 15 | schema: 16 | regex: ^([a-z\-\_]+\.)*[a-z\-\_]+$ 17 | 18 | cobbler..profile: 19 | type: string 20 | regex: ^(redhat|fedora)-[0-9]+-x86_64$ 21 | -------------------------------------------------------------------------------- /examples/host/input/test_example_com.yaml: -------------------------------------------------------------------------------- 1 | location..region: emea 2 | location..country: nl 3 | location..datacenter: foobar 4 | environment: dev 5 | roles: 6 | - salt.master 7 | network..gateway: 10.0.0.254 8 | network..interfaces..eth0..hwaddr: 00:20:26:a1:12:12 9 | network..interfaces..eth0..dhcp: False 10 | network..interfaces..eth0..ipv4: 10.0.0.3 11 | network..interfaces..eth0..netmask: 255.255.255.0 12 | network..interfaces..eth0..fqdn: {{ hostname }} 13 | cobbler..profile: fedora-19-x86_64 14 | -------------------------------------------------------------------------------- /examples/host/osfinger/fedora_19.yaml: -------------------------------------------------------------------------------- 1 | pkgrepo..type: yum 2 | pkgrepo..osabbr: fc19 3 | pkgrepo..repos..base..name: {{ osfinger }} - Base 4 | pkgrepo..repos..base..baseurl: http://{{ pkgrepo.mirror }}/pub/fedora/linux/releases/$releasever/Fedora/x86_64/os/ 5 | pkgrepo..repos..updates..name: {{ osfinger }} - Updates 6 | pkgrepo..repos..updates..baseurl: http://{{ pkgrepo.mirror }}/pub/fedora/updates/$releasever/x86_64/ 7 | pkgrepo..repos..everything..name: {{ osfinger }} - Everything 8 | pkgrepo..repos..everything..baseurl: http://{{ pkgrepo.mirror }}/pub/fedora/linux/releases/$releasever/Everything/x86_64/os/ 9 | -------------------------------------------------------------------------------- /tests/test_output.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import yaml 5 | import subprocess 6 | import sys 7 | 8 | sys.exit(0) 9 | 10 | # Validate importing Pepa 11 | 12 | # Validate calling Pepa from TTY 13 | expected = yaml.load(open('tests/output.yaml').read()) 14 | proc = subprocess.Popen(['pepa', '-n', '-c', 'examples/master', 'test.example.com'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 15 | actual = yaml.load(proc.communicate()[0]) 16 | 17 | if cmp(expected, actual) == 0: 18 | print "SUCCESS!" 19 | sys.exit(0) 20 | else: 21 | print "FAILED!" 22 | sys.exit(1) 23 | -------------------------------------------------------------------------------- /examples/host/schemas/pkgrepo.yaml: -------------------------------------------------------------------------------- 1 | {% set hostname = '^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-zA-Z]{2,6}$' %} 2 | {% set url = '(http|https?://([-\w\.]+)+(:\d+)?(/([\w/_\.]*(\?\S+)?)?)?)' %} 3 | 4 | pkgrepo..mirror: 5 | type: string 6 | regex: {{ hostname }} 7 | 8 | pkgrepo..type: 9 | type: string 10 | allowed: yum 11 | 12 | pkgrepo..osabbr: 13 | type: string 14 | regex: ^(fc|rhel)[0-9]+$ 15 | 16 | {% for repo in [ 'base', 'everything', 'updates' ] %} 17 | pkgrepo..repos..{{ repo }}..name: 18 | type: string 19 | regex: ^[A-Za-z\ 0-9\-\_]+$ 20 | 21 | pkgrepo..repos..{{ repo }}..baseurl: 22 | type: string 23 | regex: {{ url }} 24 | {% endfor %} 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Pepa - Configuration templating for SaltStack using Hierarchical substitution and Jinja 2 | 3 | Copyright (c) 2013 Michael Persson 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | -------------------------------------------------------------------------------- /examples/host/schemas/network.yaml: -------------------------------------------------------------------------------- 1 | {% set addr = '^([0-9]{1,3}\.){3}[0-9]{1,3}$' %} 2 | 3 | network..dns..options: 4 | type: list 5 | schema: 6 | regex: ^[a-z]+:[a-z0-9]+$ 7 | 8 | network..dns..search: 9 | type: list 10 | schema: 11 | regex: ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-zA-Z]{2,6}$ 12 | 13 | network..dns..servers: 14 | type: list 15 | schema: 16 | regex: {{ addr }} 17 | 18 | {% for interface in [ 'eth0' ] %} 19 | network..interfaces..{{ interface }}..ipv4: 20 | type: string 21 | regex: {{ addr }} 22 | 23 | network..interfaces..{{ interface }}..hwaddr: 24 | type: string 25 | regex: ^([0-9a-fA-F]{1,2}\:){5}[0-9a-fA-F]{1,2}$ 26 | 27 | network..interfaces..{{ interface }}..fqdn: 28 | type: string 29 | regex: ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-zA-Z]{2,6}$ 30 | 31 | network..interfaces..{{ interface }}..dhcp: 32 | type: boolean 33 | 34 | network..gateway: 35 | type: string 36 | regex: {{ addr }} 37 | 38 | network..interfaces..{{ interface }}..netmask: 39 | type: string 40 | regex: {{ addr }} 41 | {% endfor %} 42 | -------------------------------------------------------------------------------- /examples/master: -------------------------------------------------------------------------------- 1 | auto_accept: True 2 | 3 | file_roots: 4 | base: 5 | - /srv/salt/base/states 6 | qa: 7 | - /srv/salt/qa/states 8 | prod: 9 | - /srv/salt/prod/states 10 | 11 | pillar_roots: 12 | base: 13 | - /srv/salt/base/pillars 14 | qa: 15 | - /srv/salt/qa/pillars 16 | prod: 17 | - /srv/salt/prod/pillars 18 | 19 | extension_modules: /srv/salt/ext 20 | 21 | ext_pillar: 22 | - pepa: 23 | resource: host 24 | sequence: 25 | - hostname: 26 | name: input 27 | base_only: True 28 | - default: 29 | - environment: 30 | - location..region: 31 | name: region 32 | - location..country: 33 | name: country 34 | - location..datacenter: 35 | name: datacenter 36 | - roles: 37 | - osfinger: 38 | - hostname: 39 | name: override 40 | base_only: True 41 | # subkey: True 42 | # subkey_only: True 43 | 44 | pepa_grains: 45 | osfinger: Fedora-19 46 | 47 | pepa_roots: 48 | base: examples 49 | dev: examples 50 | qa: examples 51 | prod: examples 52 | 53 | #log_level: debug 54 | 55 | #log_granular_levels: 56 | # salt: warning 57 | # salt.loaded.ext.pillar.pepa: debug 58 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | Setup script for pepa 5 | ''' 6 | 7 | from setuptools import setup, find_packages 8 | import sys, os 9 | 10 | CLASSIFIERS = [ 11 | 'Development Status :: 5 - Production/Stable', 12 | 'Environment :: Console', 13 | 'Intended Audience :: System Administrators', 14 | 'License :: OSI Approved :: Apache Software License', 15 | 'Operating System :: POSIX :: Linux', 16 | ] 17 | REQUIRES = [ 18 | 'pyyaml', 19 | 'jinja2', 20 | 'argparse', 21 | 'logging', 22 | 'colorlog', 23 | 'cerberus', 24 | 'requests', 25 | ] 26 | 27 | setup( 28 | name = 'pepa', 29 | version = '0.8.1', 30 | 31 | description = 'Configuration templating for SaltStack using Hierarchical substitution and Jinja', 32 | long_description = open("README.rst").read(), 33 | 34 | author = 'Michael Persson', 35 | author_email = 'michael.ake.persson@gmail.com', 36 | url = 'https://github.com/mickep76/pepa.git', 37 | license = 'Apache License, Version 2.0', 38 | 39 | packages = find_packages(exclude=['examples', 'tests']), 40 | classifiers = CLASSIFIERS, 41 | scripts = ['scripts/pepa', 'scripts/pepa-test'], 42 | install_requires = REQUIRES, 43 | ) 44 | -------------------------------------------------------------------------------- /tests/output.yaml: -------------------------------------------------------------------------------- 1 | burgerking: 2 | burgers: 3 | - WHOPPER 4 | - DOUBLE WHOPPER 5 | cobbler: 6 | profile: fedora-19-x86_64 7 | default: default 8 | environment: dev 9 | hostname: test.example.com 10 | location: 11 | country: nl 12 | datacenter: foobar 13 | region: emea 14 | mail: 15 | agent: sendmail 16 | gateway: smtp.emea.example.com 17 | network: 18 | dns: 19 | options: 20 | - timeout:2 21 | - attempts:1 22 | - ndots:1 23 | search: 24 | - example.com 25 | servers: 26 | - 10.2.0.1 27 | - 10.2.0.2 28 | gateway: 10.0.0.254 29 | interfaces: 30 | eth0: 31 | dhcp: false 32 | fqdn: test.example.com 33 | hwaddr: 00:20:26:a1:12:12 34 | ipv4: 10.0.0.3 35 | netmask: 255.255.255.0 36 | osfinger: Fedora-19 37 | pepa_templates: 38 | - examples/hosts/host_input/test_example_com.yaml 39 | - examples/hosts/default/default.yaml 40 | - examples/hosts/region/emea.yaml 41 | - examples/hosts/osfinger/fedora_19.yaml 42 | - examples/hosts/host_override/test_example_com.yaml 43 | pkgrepo: 44 | mirror: yum.emea.example.com 45 | osabbr: fc19 46 | repos: 47 | base: 48 | baseurl: http://yum.emea.example.com/pub/fedora/linux/releases/$releasever/Fedora/x86_64/os/ 49 | name: Fedora-19 - Base 50 | everything: 51 | baseurl: http://yum.emea.example.com/pub/fedora/linux/releases/$releasever/Everything/x86_64/os/ 52 | name: Fedora-19 - Everything 53 | updates: 54 | baseurl: http://yum.emea.example.com/pub/fedora/updates/$releasever/x86_64/ 55 | name: Fedora-19 - Updates 56 | type: yum 57 | roles: 58 | - salt.master 59 | salt: 60 | master: salt.emea.example.com 61 | release: 1 62 | version: 2014.1.5 63 | time: 64 | ntp: 65 | servers: 66 | - ntp1.amer.example.com 67 | - ntp2.amer.example.com 68 | - ntp3.amer.example.com 69 | timezone: Europe/Stockholm 70 | 71 | -------------------------------------------------------------------------------- /scripts/pepa-test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python -u 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | CLI interface for Pepa Test 5 | ''' 6 | 7 | import logging 8 | import argparse 9 | from os.path import isfile 10 | import yaml 11 | import sys 12 | import pepa 13 | 14 | # Get arguments 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('-c', '--config', default='/etc/salt/master', help='Configuration file') 17 | parser.add_argument('-r', '--resource', help='Resource, defaults to first resource') 18 | parser.add_argument('-d', '--debug', action='store_true', help='Print debug info') 19 | parser.add_argument('-s', '--show', action='store_true', help='Show result of template') 20 | parser.add_argument('-t', '--teamcity', action='store_true', help='Output validation in TeamCity format') 21 | parser.add_argument('-n', '--no-color', dest='color', action='store_false', help='No color output') 22 | args = parser.parse_args() 23 | 24 | # Create formatter 25 | if args.color: 26 | try: 27 | import colorlog 28 | formatter = colorlog.ColoredFormatter("[%(log_color)s%(levelname)-8s%(reset)s] %(log_color)s%(message)s%(reset)s") 29 | except ImportError: 30 | formatter = logging.Formatter("[%(levelname)-8s] %(message)s") 31 | else: 32 | formatter = logging.Formatter("[%(levelname)-8s] %(message)s") 33 | 34 | # Create console handle 35 | console = logging.StreamHandler() 36 | console.setFormatter(formatter) 37 | 38 | loglvl = logging.WARN 39 | if args.debug: 40 | loglvl = logging.DEBUG 41 | 42 | # Create logger 43 | logger = logging.getLogger(__name__) 44 | logger.setLevel(loglvl) 45 | logger.addHandler(console) 46 | 47 | # Create logger for module 48 | logger_pepa = logging.getLogger('pepa') 49 | logger_pepa.setLevel(loglvl) 50 | logger_pepa.addHandler(console) 51 | 52 | # Load configuration file 53 | if not isfile(args.config): 54 | logger.critical("Configuration file doesn't exist: {0}".format(args.config)) 55 | sys.exit(1) 56 | 57 | conf_yaml = open(args.config).read() 58 | try: 59 | conf = yaml.load(conf_yaml) 60 | except Exception, e: 61 | logger.critical('Failed to parse YAML in config file: {0}'.format(e)) 62 | sys.exit(1) 63 | 64 | # Load configuration file 65 | if not isfile(args.config): 66 | logger.critical('Configuration file doesn\'t exist {0}'.format(args.config)) 67 | sys.exit(1) 68 | 69 | # Get resource 70 | loc = 0 71 | for name in [e.keys()[0] for e in conf['ext_pillar']]: 72 | if name == 'pepa': 73 | if args.resource is None or args.resource == conf['ext_pillar'][loc]['pepa']['resource']: 74 | break 75 | loc += 1 76 | 77 | p = conf['ext_pillar'][loc]['pepa'] 78 | 79 | # Test templates 80 | p = conf['ext_pillar'][loc]['pepa'] 81 | templ = pepa.Template(roots=conf['pepa_roots'], resource=p['resource'], sequence=p['sequence']) 82 | success = templ.test(show=args.show, teamcity=args.teamcity) 83 | if not success: 84 | sys.exit(1) 85 | -------------------------------------------------------------------------------- /scripts/pepa: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python -u 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | CLI interface for Pepa 5 | ''' 6 | 7 | import argparse 8 | import sys 9 | from os.path import isfile 10 | import yaml 11 | import logging 12 | import pepa 13 | import requests 14 | import getpass 15 | 16 | # Add show option 17 | 18 | # Get arguments 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument('hostname', help='Hostname') 21 | parser.add_argument('-c', '--config', default='/etc/salt/master', help='Configuration file') 22 | parser.add_argument('-r', '--resource', help='Resource, defaults to first resource') 23 | parser.add_argument('-d', '--debug', action='store_true', help='Print debug info') 24 | parser.add_argument('-g', '--grains', help='Input Grains as YAML') 25 | parser.add_argument('-p', '--pillar', help='Input Pillar as YAML') 26 | parser.add_argument('-n', '--no-color', dest='color', action='store_false', help='No color output') 27 | parser.add_argument('-q', '--query-api', action='store_true', help='Query Saltstack REST API for Grains') 28 | parser.add_argument('-u', '--url', default='https://salt:8000', help='URL for SaltStack REST API') 29 | parser.add_argument('-U', '--username', help='Username for SaltStack REST API') 30 | parser.add_argument('-P', '--password', help='Password for SaltStack REST API') 31 | args = parser.parse_args() 32 | 33 | # Create formatter 34 | if args.color: 35 | try: 36 | import colorlog 37 | formatter = colorlog.ColoredFormatter("[%(log_color)s%(levelname)-8s%(reset)s] %(log_color)s%(message)s%(reset)s") 38 | except ImportError: 39 | formatter = logging.Formatter("[%(levelname)-8s] %(message)s") 40 | else: 41 | formatter = logging.Formatter("[%(levelname)-8s] %(message)s") 42 | 43 | # Create console handle 44 | console = logging.StreamHandler() 45 | console.setFormatter(formatter) 46 | 47 | loglvl = logging.WARN 48 | if args.debug: 49 | loglvl = logging.DEBUG 50 | 51 | # Create logger 52 | logger = logging.getLogger(__name__) 53 | logger.setLevel(loglvl) 54 | logger.addHandler(console) 55 | 56 | # Create logger for module 57 | logger_pepa = logging.getLogger('pepa') 58 | logger_pepa.setLevel(loglvl) 59 | logger_pepa.addHandler(console) 60 | 61 | # Load configuration file 62 | if not isfile(args.config): 63 | logger.critical("Configuration file doesn't exist: {0}".format(args.config)) 64 | sys.exit(1) 65 | 66 | conf_yaml = open(args.config).read() 67 | try: 68 | conf = yaml.load(conf_yaml) 69 | except Exception, e: 70 | logger.critical('Failed to parse YAML in config file: {0}'.format(e)) 71 | sys.exit(1) 72 | 73 | # Get grains 74 | grains = {} 75 | if 'pepa_grains' in conf: 76 | grains = conf['pepa_grains'] 77 | if args.grains: 78 | grains.update(yaml.load(args.grains)) 79 | 80 | # Get grains from SaltStack API 81 | if args.query_api: 82 | username = args.username 83 | password = args.password 84 | if username is None: 85 | username = raw_input('Username: ') 86 | if password is None: 87 | password = getpass.getpass() 88 | 89 | logger.info('Authenticate REST API') 90 | auth = {'username': username, 'password': password, 'eauth': 'pam'} 91 | request = requests.post(args.url + '/login', auth) 92 | 93 | if not request.ok: 94 | raise RuntimeError('Failed to authenticate to SaltStack REST API: {0}'.format(request.text)) 95 | 96 | response = request.json() 97 | token = response['return'][0]['token'] 98 | 99 | logger.info('Request Grains from REST API') 100 | headers = {'X-Auth-Token': token, 'Accept': 'application/json'} 101 | request = requests.get(args.url + '/minions/' + args.hostname, headers=headers) 102 | 103 | result = request.json().get('return', [{}])[0] 104 | if args.hostname not in result: 105 | raise RuntimeError('Failed to get Grains from SaltStack REST API') 106 | 107 | grains.update(result[args.hostname]) 108 | 109 | # Get pillar 110 | pillar = {} 111 | if 'pepa_pillar' in conf: 112 | pillar = conf['pepa_pillar'] 113 | if args.pillar: 114 | pillar.update(yaml.load(args.pillar)) 115 | 116 | # Get resource 117 | loc = 0 118 | for name in [e.keys()[0] for e in conf['ext_pillar']]: 119 | if name == 'pepa': 120 | if args.resource is None or args.resource == conf['ext_pillar'][loc]['pepa']['resource']: 121 | break 122 | loc += 1 123 | 124 | p = conf['ext_pillar'][loc]['pepa'] 125 | 126 | # Subkey 127 | subkey = False 128 | if 'subkey' in p: 129 | subkey = p['subkey'] 130 | 131 | subkey_only = False 132 | if 'subkey_only' in p: 133 | subkey_only = p['subkey_only'] 134 | 135 | # Compile templates 136 | templ = pepa.Template(roots=conf['pepa_roots'], resource=p['resource'], sequence=p['sequence'], subkey=subkey, subkey_only=subkey_only) 137 | res = templ.compile(minion_id=args.hostname, grains=grains, pillar=pillar) 138 | 139 | # Print result 140 | yaml.dumper.SafeDumper.ignore_aliases = lambda self, data: True 141 | print yaml.safe_dump(res) 142 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Pepa 2 | ==== 3 | 4 | Configuration templating for SaltStack using Hierarchical substitution and Jinja. 5 | 6 | .. image:: https://drone.io/github.com/mickep76/pepa/status.png 7 | :alt: drone.io build status 8 | :target: https://drone.io/github.com/mickep76/pepa 9 | 10 | Pepa is part of the SaltStack as of release 2014.7. 11 | 12 | Quick testing 13 | ============= 14 | 15 | You can easily test Pepa from the Command Line. 16 | 17 | Create a virtual env. and install the required modules. 18 | 19 | .. code-block:: bash 20 | 21 | virtualenv venv 22 | cd venv 23 | source bin/activate 24 | pip install pepa 25 | 26 | Clone and run Pepa. 27 | 28 | .. code-block:: bash 29 | 30 | git clone https://github.com/mickep76/pepa.git 31 | cd pepa 32 | pepa -c examples/master test.example.com -d 33 | 34 | Test and validate templates. 35 | 36 | .. code-block:: bash 37 | 38 | pepa-test --config examples/master -d 39 | 40 | Look at output. 41 | 42 | .. code-block:: bash 43 | 44 | pepa-test --config examples/master -d -s 45 | 46 | Install Pepa 47 | ============ 48 | 49 | .. code-block:: bash 50 | 51 | git clone https://github.com/mickep76/pepa.git 52 | mkdir -p /srv/salt/ext/pillar 53 | cp pillar/pepa.py /srv/salt/ext/pillar/pepa.py 54 | 55 | Configuring Pepa 56 | ================ 57 | 58 | .. code-block:: yaml 59 | 60 | extension_modules: /srv/salt/ext 61 | 62 | ext_pillar: 63 | - pepa: 64 | resource: host # Name of resource directory and sub-key in pillars 65 | sequence: # Sequence used for hierarchical substitution 66 | - hostname: # Name of key 67 | name: input # Alias used for template directory 68 | base_only: True # Only use templates from Base environment, i.e. no staging 69 | - default: 70 | - environment: 71 | - location..region: 72 | name: region 73 | - location..country: 74 | name: country 75 | - location..datacenter: 76 | name: datacenter 77 | - roles: 78 | - osfinger: 79 | name: os 80 | - hostname: 81 | name: override 82 | base_only: True 83 | subkey: True # Create a sub-key in pillars, named after the resource in this case [host] 84 | subkey_only: True # Only create a sub-key, and leave the top level untouched 85 | 86 | pepa_roots: # Base directory for each environment 87 | base: /srv/pepa/base # Path for base environment 88 | dev: /srv/pepa/base # Associate dev with base 89 | qa: /srv/pepa/qa 90 | prod: /srv/pepa/prod 91 | 92 | # Use a different delimiter for nested dictionaries, defaults to '..' since some keys may use '.' in the name 93 | #pepa_delimiter: .. 94 | 95 | # Supply Grains for Pepa, this should **ONLY** be used for testing or validation 96 | #pepa_grains: 97 | # environment: dev 98 | 99 | # Supply Pillar for Pepa, this should **ONLY** be used for testing or validation 100 | #pepa_pillars: 101 | # saltversion: 0.17.4 102 | 103 | # Enable debug for Pepa, and keep Salt on warning 104 | #log_level: debug 105 | 106 | #log_granular_levels: 107 | # salt: warning 108 | # salt.loaded.ext.pillar.pepa: debug 109 | 110 | Pepa can also be used in Master-less SaltStack setup. 111 | 112 | Command line 113 | ============ 114 | 115 | .. code-block:: bash 116 | 117 | usage: pepa [-h] [-c CONFIG] [-d] [-g GRAINS] [-p PILLAR] [-n] [-v] 118 | hostname 119 | 120 | positional arguments: 121 | hostname Hostname 122 | 123 | optional arguments: 124 | -h, --help show this help message and exit 125 | -c CONFIG, --config CONFIG 126 | Configuration file 127 | -r RESOURCE, --resource RESOURCE 128 | Resource, defaults to first resource 129 | -d, --debug Print debug info 130 | -g GRAINS, --grains GRAINS 131 | Input Grains as YAML 132 | -p PILLAR, --pillar PILLAR 133 | Input Pillar as YAML 134 | -n, --no-color No color output 135 | -v, --validate Validate output 136 | 137 | Templates 138 | ========= 139 | 140 | Templates is configuration for a host or software, that can use information from Grains or Pillars. These can then be used for hierarchically substitution. 141 | 142 | **Example File:** host/input/test_example_com.yaml 143 | 144 | .. code-block:: yaml 145 | 146 | location..region: emea 147 | location..country: nl 148 | location..datacenter: foobar 149 | environment: dev 150 | roles: 151 | - salt.master 152 | network..gateway: 10.0.0.254 153 | network..interfaces..eth0..hwaddr: 00:20:26:a1:12:12 154 | network..interfaces..eth0..dhcp: False 155 | network..interfaces..eth0..ipv4: 10.0.0.3 156 | network..interfaces..eth0..netmask: 255.255.255.0 157 | network..interfaces..eth0..fqdn: {{ hostname }} 158 | cobbler..profile: fedora-19-x86_64 159 | 160 | As you see in this example you can use Jinja directly inside the template. 161 | 162 | **Example File:** host/region/amer.yaml 163 | 164 | .. code-block:: yaml 165 | 166 | network..dns..servers: 167 | - 10.0.0.1 168 | - 10.0.0.2 169 | time..ntp..servers: 170 | - ntp1.amer.example.com 171 | - ntp2.amer.example.com 172 | - ntp3.amer.example.com 173 | time..timezone: America/Chihuahua 174 | yum..mirror: yum.amer.example.com 175 | 176 | Each template is named after the value of the key using lowercase and all extended characters are replaced with underscore. 177 | 178 | **Example:** 179 | 180 | osfinger: Fedora-19 181 | 182 | **Would become:** 183 | 184 | fedora_19.yaml 185 | 186 | Nested dictionaries 187 | =================== 188 | 189 | In order to create nested dictionaries as output you can use double dot **".."** as a delimiter. You can change this using "pepa_delimiter" we choose double dot since single dot is already used by key names in some modules, and using ":" requires quoting in the YAML. 190 | 191 | **Example:** 192 | 193 | .. code-block:: yaml 194 | 195 | network..dns..servers: 196 | - 10.0.0.1 197 | - 10.0.0.2 198 | network..dns..options: 199 | - timeout:2 200 | - attempts:1 201 | - ndots:1 202 | network..dns..search: 203 | - example.com 204 | 205 | **Would become:** 206 | 207 | .. code-block:: yaml 208 | 209 | network: 210 | dns: 211 | servers: 212 | - 10.0.0.1 213 | - 10.0.0.2 214 | options: 215 | - timeout:2 216 | - attempts:1 217 | - ndots:1 218 | search: 219 | - example.com 220 | 221 | Operators 222 | ========= 223 | 224 | Operators can be used to merge/unset a list/hash or set the key as immutable, so it can't be changed. 225 | 226 | =========== ================================================ 227 | Operator Description 228 | =========== ================================================ 229 | merge() Merge list or hash 230 | unset() Unset key 231 | immutable() Set the key as immutable, so it can't be changed 232 | imerge() Set immutable and merge 233 | iunset() Set immutable and unset 234 | =========== ================================================ 235 | 236 | **Example:** 237 | 238 | .. code-block:: yaml 239 | 240 | network..dns..search..merge(): 241 | - foobar.com 242 | - dummy.nl 243 | owner..immutable(): Operations 244 | host..printers..unset(): 245 | 246 | Testing 247 | ======= 248 | 249 | Pepa also come's with a test/validation tool for templates. This allows you to test for valid Jinja/YAML and validate key values. 250 | 251 | Command Line 252 | ============ 253 | 254 | .. code-block:: bash 255 | 256 | usage: pepa-test [-h] [-c CONFIG] [-r RESOURCE] [-d] [-s] [-t] [-n] 257 | 258 | optional arguments: 259 | -h, --help show this help message and exit 260 | -c CONFIG, --config CONFIG 261 | Configuration file 262 | -r RESOURCE, --resource RESOURCE 263 | Configuration file, defaults to first resource 264 | -d, --debug Print debug info 265 | -s, --show Show result of template 266 | -t, --teamcity Output validation in TeamCity format 267 | -n, --no-color No color output 268 | 269 | Test 270 | ==== 271 | 272 | A test is a set of input values for a template, it's generally a good idea to create a separate test for each outcome if you have Jinja if statements. 273 | 274 | **Example:** host/default/tests/default-1.yaml 275 | 276 | .. code-block:: yaml 277 | 278 | grains..osfinger: Fedora-20 279 | location..region: emea 280 | 281 | You can also use Jinja inside a test, for example if you wan't to iterate through test values. 282 | 283 | Schema 284 | ====== 285 | 286 | A schema is a set of validation rules for each key/value. Schemas use `Cerberus `_ module for validation. 287 | 288 | **Example:** host/schemas/pkgrepo.yaml 289 | 290 | .. code-block:: yaml 291 | 292 | {% set hostname = '^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-zA-Z]{2,6}$' %} 293 | {% set url = '(http|https?://([-\w\.]+)+(:\d+)?(/([\w/_\.]*(\?\S+)?)?)?)' %} 294 | 295 | pkgrepo..mirror: 296 | type: string 297 | regex: {{ hostname }} 298 | 299 | pkgrepo..type: 300 | type: string 301 | allowed: yum 302 | 303 | pkgrepo..osabbr: 304 | type: string 305 | regex: ^(fc|rhel)[0-9]+$ 306 | 307 | {% for repo in [ 'base', 'everything', 'updates' ] %} 308 | pkgrepo..repos..{{ repo }}..name: 309 | type: string 310 | regex: ^[A-Za-z\ 0-9\-\_]+$ 311 | 312 | pkgrepo..repos..{{ repo }}..baseurl: 313 | type: string 314 | regex: {{ url }} 315 | {% endfor %} 316 | 317 | You can also use Jinja inside a schema, for example if you wan't to iterate through a list of different keys. 318 | 319 | You can create complicated datastructures underneth a key, but it's advisable to split it in several 320 | keys using the delimiter for a nested data structures. 321 | 322 | **Bad** 323 | 324 | .. code-block:: yaml 325 | 326 | network: 327 | interfaces: 328 | eth0: 329 | ipv4: 192.168.1.2 330 | netmask: 255.255.255.0 331 | 332 | **Good** 333 | 334 | .. code-block:: yaml 335 | 336 | network..interfaces..eth0..ipv4: 192.168.1.2 337 | network..interfaces..eth0..netmask: 255.255.255.0 338 | 339 | The first example you can't properly use substitution and defining the schema becomes more complicated. 340 | -------------------------------------------------------------------------------- /pillar/pepa.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python -u 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | Pepa 5 | ==== 6 | 7 | Configuration templating for SaltStack using Hierarchical substitution and Jinja. 8 | 9 | Configuring Pepa 10 | ================ 11 | 12 | .. code-block:: yaml 13 | 14 | extension_modules: /srv/salt/ext 15 | 16 | ext_pillar: 17 | - pepa: 18 | resource: host # Name of resource directory and sub-key in pillars 19 | sequence: # Sequence used for hierarchical substitution 20 | - hostname: # Name of key 21 | name: input # Alias used for template directory 22 | base_only: True # Only use templates from Base environment, i.e. no staging 23 | - default: 24 | - environment: 25 | - location..region: 26 | name: region 27 | - location..country: 28 | name: country 29 | - location..datacenter: 30 | name: datacenter 31 | - roles: 32 | - osfinger: 33 | name: os 34 | - hostname: 35 | name: override 36 | base_only: True 37 | subkey: True # Create a sub-key in pillars, named after the resource in this case [host] 38 | subkey_only: True # Only create a sub-key, and leave the top level untouched 39 | 40 | pepa_roots: # Base directory for each environment 41 | base: /srv/pepa/base # Path for base environment 42 | dev: /srv/pepa/base # Associate dev with base 43 | qa: /srv/pepa/qa 44 | prod: /srv/pepa/prod 45 | 46 | # Use a different delimiter for nested dictionaries, defaults to '..' since some keys may use '.' in the name 47 | #pepa_delimiter: .. 48 | 49 | # Supply Grains for Pepa, this should **ONLY** be used for testing or validation 50 | #pepa_grains: 51 | # environment: dev 52 | 53 | # Supply Pillar for Pepa, this should **ONLY** be used for testing or validation 54 | #pepa_pillars: 55 | # saltversion: 0.17.4 56 | 57 | # Enable debug for Pepa, and keep Salt on warning 58 | #log_level: debug 59 | 60 | #log_granular_levels: 61 | # salt: warning 62 | # salt.loaded.ext.pillar.pepa: debug 63 | 64 | Pepa can also be used in Master-less SaltStack setup. 65 | 66 | Templates 67 | ========= 68 | 69 | Templates is configuration for a host or software, that can use information from Grains or Pillars. These can then be used for hierarchically substitution. 70 | 71 | **Example File:** host/input/test_example_com.yaml 72 | 73 | .. code-block:: yaml 74 | 75 | location..region: emea 76 | location..country: nl 77 | location..datacenter: foobar 78 | environment: dev 79 | roles: 80 | - salt.master 81 | network..gateway: 10.0.0.254 82 | network..interfaces..eth0..hwaddr: 00:20:26:a1:12:12 83 | network..interfaces..eth0..dhcp: False 84 | network..interfaces..eth0..ipv4: 10.0.0.3 85 | network..interfaces..eth0..netmask: 255.255.255.0 86 | network..interfaces..eth0..fqdn: {{ hostname }} 87 | cobbler..profile: fedora-19-x86_64 88 | 89 | As you see in this example you can use Jinja directly inside the template. 90 | 91 | **Example File:** host/region/amer.yaml 92 | 93 | .. code-block:: yaml 94 | 95 | network..dns..servers: 96 | - 10.0.0.1 97 | - 10.0.0.2 98 | time..ntp..servers: 99 | - ntp1.amer.example.com 100 | - ntp2.amer.example.com 101 | - ntp3.amer.example.com 102 | time..timezone: America/Chihuahua 103 | yum..mirror: yum.amer.example.com 104 | 105 | Each template is named after the value of the key using lowercase and all extended characters are replaced with underscore. 106 | 107 | **Example:** 108 | 109 | osfinger: Fedora-19 110 | 111 | **Would become:** 112 | 113 | fedora_19.yaml 114 | 115 | Nested dictionaries 116 | =================== 117 | 118 | In order to create nested dictionaries as output you can use double dot **".."** as a delimiter. You can change this using "pepa_delimiter" we choose double dot since single dot is already used by key names in some modules, and using ":" requires quoting in the YAML. 119 | 120 | **Example:** 121 | 122 | .. code-block:: yaml 123 | 124 | network..dns..servers: 125 | - 10.0.0.1 126 | - 10.0.0.2 127 | network..dns..options: 128 | - timeout:2 129 | - attempts:1 130 | - ndots:1 131 | network..dns..search: 132 | - example.com 133 | 134 | **Would become:** 135 | 136 | .. code-block:: yaml 137 | 138 | network: 139 | dns: 140 | servers: 141 | - 10.0.0.1 142 | - 10.0.0.2 143 | options: 144 | - timeout:2 145 | - attempts:1 146 | - ndots:1 147 | search: 148 | - example.com 149 | 150 | Operators 151 | ========= 152 | 153 | Operators can be used to merge/unset a list/hash or set the key as immutable, so it can't be changed. 154 | 155 | =========== ================================================ 156 | Operator Description 157 | =========== ================================================ 158 | merge() Merge list or hash 159 | unset() Unset key 160 | immutable() Set the key as immutable, so it can't be changed 161 | imerge() Set immutable and merge 162 | iunset() Set immutable and unset 163 | =========== ================================================ 164 | 165 | **Example:** 166 | 167 | .. code-block:: yaml 168 | 169 | network..dns..search..merge(): 170 | - foobar.com 171 | - dummy.nl 172 | owner..immutable(): Operations 173 | host..printers..unset(): 174 | 175 | Links 176 | ===== 177 | 178 | For more examples and information see . 179 | ''' 180 | 181 | __author__ = 'Michael Persson ' 182 | __author_email__ = 'michael.ake.persson@gmail.com' 183 | __copyright__ = 'Copyright (c) 2013 Michael Persson' 184 | __license__ = 'Apache License, Version 2.0' 185 | __version__ = '0.7.6' 186 | __url__ = 'https://github.com/mickep76/pepa.git' 187 | 188 | # Import python libs 189 | import logging 190 | import sys 191 | import glob 192 | import yaml 193 | import jinja2 194 | import re 195 | from os.path import isfile, join 196 | 197 | # Options 198 | __opts__ = { 199 | 'pepa_roots': { 200 | 'base': '/srv/salt' 201 | }, 202 | 'pepa_delimiter': '..', 203 | 'pepa_validate': False 204 | } 205 | 206 | # Set up logging 207 | log = logging.getLogger(__name__) 208 | 209 | 210 | def key_value_to_tree(data): 211 | ''' 212 | Convert key/value to tree 213 | ''' 214 | tree = {} 215 | for flatkey, value in data.items(): 216 | t = tree 217 | keys = flatkey.split(__opts__['pepa_delimiter']) 218 | for key in keys: 219 | if key == keys[-1]: 220 | t[key] = value 221 | else: 222 | t = t.setdefault(key, {}) 223 | return tree 224 | 225 | 226 | def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only=False): 227 | ''' 228 | Evaluate Pepa templates 229 | ''' 230 | roots = __opts__['pepa_roots'] 231 | 232 | # Default input 233 | inp = {} 234 | inp['default'] = 'default' 235 | inp['hostname'] = minion_id 236 | 237 | if 'environment' in pillar: 238 | inp['environment'] = pillar['environment'] 239 | elif 'environment' in __grains__: 240 | inp['environment'] = __grains__['environment'] 241 | else: 242 | inp['environment'] = 'base' 243 | 244 | # Load templates 245 | output = inp 246 | output['pepa_templates'] = [] 247 | immutable = {} 248 | 249 | for categ, info in [s.items()[0] for s in sequence]: 250 | if categ not in inp: 251 | log.warn("Category is not defined: {0}".format(categ)) 252 | continue 253 | 254 | alias = None 255 | if isinstance(info, dict) and 'name' in info: 256 | alias = info['name'] 257 | else: 258 | alias = categ 259 | 260 | templdir = None 261 | if info and 'base_only' in info and info['base_only']: 262 | templdir = join(roots['base'], resource, alias) 263 | else: 264 | templdir = join(roots[inp['environment']], resource, alias) 265 | 266 | entries = [] 267 | if isinstance(inp[categ], list): 268 | entries = inp[categ] 269 | elif not inp[categ]: 270 | log.warn("Category has no value set: {0}".format(categ)) 271 | continue 272 | else: 273 | entries = [inp[categ]] 274 | 275 | for entry in entries: 276 | results_jinja = None 277 | results = None 278 | fn = join(templdir, re.sub(r'\W', '_', entry.lower()) + '.yaml') 279 | if isfile(fn): 280 | log.info("Loading template: {0}".format(fn)) 281 | template = jinja2.Template(open(fn).read()) 282 | output['pepa_templates'].append(fn) 283 | 284 | try: 285 | data = key_value_to_tree(output) 286 | data['grains'] = __grains__.copy() 287 | data['pillar'] = pillar.copy() 288 | results_jinja = template.render(data) 289 | results = yaml.load(results_jinja) 290 | except jinja2.UndefinedError, err: 291 | log.error('Failed to parse JINJA template: {0}\n{1}'.format(fn, err)) 292 | except yaml.YAMLError, err: 293 | log.error('Failed to parse YAML in template: {0}\n{1}'.format(fn, err)) 294 | else: 295 | log.info("Template doesn't exist: {0}".format(fn)) 296 | continue 297 | 298 | if results is not None: 299 | for key in results: 300 | skey = key.rsplit(__opts__['pepa_delimiter'], 1) 301 | rkey = None 302 | operator = None 303 | if len(skey) > 1 and key.rfind('()') > 0: 304 | rkey = skey[0].rstrip(__opts__['pepa_delimiter']) 305 | operator = skey[1] 306 | 307 | if key in immutable: 308 | log.warning('Key {0} is immutable, changes are not allowed'.format(key)) 309 | elif rkey in immutable: 310 | log.warning("Key {0} is immutable, changes are not allowed".format(rkey)) 311 | elif operator == 'merge()' or operator == 'imerge()': 312 | if operator == 'merge()': 313 | log.debug("Merge key {0}: {1}".format(rkey, results[key])) 314 | else: 315 | log.debug("Set immutable and merge key {0}: {1}".format(rkey, results[key])) 316 | immutable[rkey] = True 317 | if rkey not in output: 318 | log.error('Cant\'t merge key {0} doesn\'t exist'.format(rkey)) 319 | elif type(results[key]) != type(output[rkey]): 320 | log.error('Can\'t merge different types for key {0}'.format(rkey)) 321 | elif type(results[key]) is dict: 322 | output[rkey].update(results[key]) 323 | elif type(results[key]) is list: 324 | output[rkey].extend(results[key]) 325 | else: 326 | log.error('Unsupported type need to be list or dict for key {0}'.format(rkey)) 327 | elif operator == 'unset()' or operator == 'iunset()': 328 | if operator == 'unset()': 329 | log.debug("Unset key {0}".format(rkey)) 330 | else: 331 | log.debug("Set immutable and unset key {0}".format(rkey)) 332 | immutable[rkey] = True 333 | if rkey in output: 334 | del output[rkey] 335 | elif operator == 'immutable()': 336 | log.debug("Set immutable and substitute key {0}: {1}".format(rkey, results[key])) 337 | immutable[rkey] = True 338 | output[rkey] = results[key] 339 | elif operator is not None: 340 | log.error('Unsupported operator {0}, skipping key {1}'.format(operator, rkey)) 341 | else: 342 | log.debug("Substitute key {0}: {1}".format(key, results[key])) 343 | output[key] = results[key] 344 | 345 | tree = key_value_to_tree(output) 346 | pillar_data = {} 347 | if subkey_only: 348 | pillar_data[resource] = tree.copy() 349 | elif subkey: 350 | pillar_data = tree 351 | pillar_data[resource] = tree.copy() 352 | else: 353 | pillar_data = tree 354 | if __opts__['pepa_validate']: 355 | pillar_data['pepa_keys'] = output.copy() 356 | return pillar_data 357 | -------------------------------------------------------------------------------- /pepa/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Configuration templating for SaltStack using Hierarchical substitution and Jinja 4 | ''' 5 | 6 | # Import python libs 7 | import sys 8 | import glob 9 | import yaml 10 | import jinja2 11 | import re 12 | from os.path import isfile, isdir, join, dirname, basename 13 | import logging 14 | import cerberus 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | def key_value_to_tree(data, delimiter): 19 | ''' 20 | Convert key/value to tree 21 | ''' 22 | tree = {} 23 | for flatkey, value in data.items(): 24 | t = tree 25 | keys = flatkey.split(delimiter) 26 | for key in keys: 27 | if key == keys[-1]: 28 | t[key] = value 29 | else: 30 | t = t.setdefault(key, {}) 31 | return tree 32 | 33 | class Template(object): 34 | ''' 35 | Template class 36 | ''' 37 | def __init__(self, roots={'base': '/srv/pepa'}, delimiter='..', resource='host', sequence={'hostname': {'name': 'input', 'base_only': True}}, subkey=False, subkey_only=False): 38 | ''' 39 | Initialize template object 40 | ''' 41 | self.roots = roots 42 | self.delimiter = delimiter 43 | self.resource = resource 44 | self.sequence = sequence 45 | self.subkey = subkey 46 | self.subkey_only = subkey_only 47 | 48 | def compile(self, minion_id, grains={}, pillar={}): 49 | ''' 50 | Compile templates 51 | ''' 52 | 53 | # Default 54 | output = {} 55 | output['default'] = 'default' 56 | output['hostname'] = minion_id 57 | 58 | # Environment 59 | if 'environment' in pillar: 60 | output['environment'] = pillar['environment'] 61 | elif 'environment' in grains: 62 | output['environment'] = grains['environment'] 63 | else: 64 | output['environment'] = 'base' 65 | 66 | immutable = {} 67 | 68 | for categ, cdata in [s.items()[0] for s in self.sequence]: 69 | if categ not in output: 70 | logger.warn("Category is not defined: {0}".format(categ)) 71 | continue 72 | 73 | # Category alias 74 | calias = categ 75 | if isinstance(cdata, dict) and 'name' in cdata: 76 | calias = cdata['name'] 77 | 78 | # Template dir. 79 | tdir = join(self.roots[output['environment']], self.resource, calias) 80 | if cdata and 'base_only' in cdata and cdata['base_only']: 81 | tdir = join(self.roots['base'], self.resource, calias) 82 | 83 | entries = [] 84 | if isinstance(output[categ], list): 85 | entries = output[categ] 86 | elif not output[categ]: 87 | logger.warn("Category has no value set: {0}".format(categ)) 88 | continue 89 | else: 90 | entries = [output[categ]] 91 | 92 | for entry in entries: 93 | fn = join(tdir, re.sub(r'\W', '_', entry.lower()) + '.yaml') 94 | if not isfile(fn): 95 | logger.info("Template doesn't exist: {0}".format(fn)) 96 | continue 97 | 98 | logger.info("Loading template: {0}".format(fn)) 99 | template = jinja2.Template(open(fn).read()) 100 | 101 | inp = key_value_to_tree(output, self.delimiter) 102 | inp['grains'] = grains.copy() 103 | inp['pillar'] = pillar.copy() 104 | try: 105 | res_jinja = template.render(inp) 106 | except Exception, e: 107 | logger.error('Failed to parse JINJA in template: {0}\n{1}'.format(fn, e)) 108 | return {} 109 | 110 | try: 111 | results = yaml.load(res_jinja) 112 | except Exception, e: 113 | logger.error('Failed to parse YAML in template: {0}\n{1}'.format(fn, e)) 114 | return {} 115 | 116 | if not results: 117 | continue 118 | 119 | for key in results: 120 | skey = key.rsplit(self.delimiter, 1) 121 | rkey = None 122 | operator = None 123 | if len(skey) > 1 and key.rfind('()') > 0: 124 | rkey = skey[0].rstrip(self.delimiter) 125 | operator = skey[1] 126 | 127 | if key in immutable: 128 | logger.warning('Key {0} is immutable, changes are not allowed'.format(key)) 129 | elif rkey in immutable: 130 | logger.warning("Key {0} is immutable, changes are not allowed".format(rkey)) 131 | elif operator == 'merge()' or operator == 'imerge()': 132 | if operator == 'merge()': 133 | logger.debug("Merge key {0}: {1}".format(rkey, results[key])) 134 | else: 135 | logger.debug("Set immutable and merge key {0}: {1}".format(rkey, results[key])) 136 | immutable[rkey] = True 137 | if rkey not in output: 138 | logger.error('Cant\'t merge key {0} doesn\'t exist'.format(rkey)) 139 | elif type(results[key]) != type(output[rkey]): 140 | logger.error('Can\'t merge different types for key {0}'.format(rkey)) 141 | elif type(results[key]) is dict: 142 | output[rkey].update(results[key]) 143 | elif type(results[key]) is list: 144 | output[rkey].extend(results[key]) 145 | else: 146 | logger.error('Unsupported type need to be list or dict for key {0}'.format(rkey)) 147 | elif operator == 'unset()' or operator == 'iunset()': 148 | if operator == 'unset()': 149 | logger.debug("Unset key {0}".format(rkey)) 150 | else: 151 | logger.debug("Set immutable and unset key {0}".format(rkey)) 152 | immutable[rkey] = True 153 | if rkey in output: 154 | del output[rkey] 155 | elif operator == 'immutable()': 156 | logger.debug("Set immutable and substitute key {0}: {1}".format(rkey, results[key])) 157 | immutable[rkey] = True 158 | output[rkey] = results[key] 159 | elif operator is not None: 160 | logger.error('Unsupported operator {0}, skipping key {1}'.format(operator, rkey)) 161 | else: 162 | logger.debug("Substitute key {0}: {1}".format(key, results[key])) 163 | output[key] = results[key] 164 | 165 | tree = key_value_to_tree(output, self.delimiter) 166 | pdata = {} 167 | if self.subkey_only: 168 | pdata[self.resource] = tree.copy() 169 | elif self.subkey: 170 | pdata = tree 171 | pdata[self.resource] = tree.copy() 172 | else: 173 | pdata = tree 174 | return pdata 175 | 176 | def test(self, show=False, teamcity=False): 177 | ''' 178 | Test templates 179 | ''' 180 | 181 | if teamcity: 182 | print "##teamcity[testSuiteStarted name='pepa']" 183 | 184 | success = True 185 | resdir = join(self.roots['base'], self.resource) 186 | schema = {} 187 | for fn in glob.glob(resdir + '/schemas/*.yaml'): 188 | sfn = 'schemas/' + basename(fn) 189 | logger.debug('Load schema {0}'.format(sfn)) 190 | 191 | template = jinja2.Template(open(fn).read()) 192 | try: 193 | res_jinja = template.render() 194 | except Exception, e: 195 | logger.critical('Failed to parse YAML in schema {0}\n{1}'.format(sfn, e)) 196 | sys.exit(1) 197 | try: 198 | res_yaml = yaml.load(res_jinja) 199 | except Exception, e: 200 | logger.critical('Failed to parse YAML in test {0}\n{1}'.format(sfn, e)) 201 | sys.exit(1) 202 | schema.update(res_yaml) 203 | 204 | if show: 205 | print '### Schema: {0} ###\n'.format(sfn) 206 | print yaml.safe_dump(res_yaml, default_flow_style=False) 207 | 208 | for categ, info in [s.items()[0] for s in self.sequence]: 209 | templdir = join(self.roots['base'], self.resource, categ) 210 | alias = categ 211 | if isinstance(info, dict) and 'name' in info: 212 | alias = info['name'] 213 | templdir = join(self.roots['base'], self.resource, alias) 214 | 215 | if not isdir(templdir + '/tests'): 216 | success = False 217 | if teamcity: 218 | print "##teamcity[testFailed name='pepa' message='No tests defined for category {0}']".format(alias) 219 | else: 220 | logger.error('No tests defined for category {0}'.format(alias)) 221 | continue 222 | 223 | for testf in glob.glob(templdir + '/tests/*.yaml'): 224 | stestf = alias + '/tests/' + basename(testf) 225 | logger.debug('Load test {0}'.format(stestf)) 226 | 227 | # Load tests 228 | template = jinja2.Template(open(testf).read()) 229 | try: 230 | res_jinja = template.render() 231 | except Exception, e: 232 | logger.critical('Failed to parse Jinja test {0}\n{1}'.format(stestf, e)) 233 | sys.exit(1) 234 | try: 235 | res_yaml = yaml.load(res_jinja) 236 | except Exception, e: 237 | logger.critical('Failed to parse YAML in test {0}\n{1}'.format(stestf, e)) 238 | sys.exit(1) 239 | 240 | if show: 241 | print '### Test: {0} ###\n'.format(stestf) 242 | print yaml.safe_dump(res_yaml, default_flow_style=False) 243 | 244 | defaults = key_value_to_tree(res_yaml, self.delimiter) 245 | 246 | for fn in glob.glob(templdir + '/*.yaml'): 247 | sfn = alias + '/' + basename(fn) 248 | 249 | logger.debug('Load template {0}'.format(sfn)) 250 | if teamcity: 251 | print "##teamcity[testStarted name='{0}']".format(sfn) 252 | 253 | # Parse Jinja 254 | template = jinja2.Template(open(fn).read()) 255 | res_jinja = None 256 | res_yaml = None 257 | try: 258 | res_jinja = template.render(defaults) 259 | except Exception, e: 260 | success = False 261 | if teamcity: 262 | print "##teamcity[testFailed name='{0}' message='Failed to parse Jinja: {1}']".format(sfn, e) 263 | print "##teamcity[testFinished name='{0}']".format(sfn) 264 | else: 265 | logger.critical('Failed to parse Jinja template {0}\n{1}'.format(sfn, e)) 266 | continue 267 | 268 | # Parse YAML 269 | try: 270 | res_yaml = yaml.load(res_jinja) 271 | except Exception, e: 272 | success = False 273 | if teamcity: 274 | print "##teamcity[testFailed name='{0}' message='Failed to parse YAML: {1}']".format(sfn, e) 275 | print "##teamcity[testFinished name='{0}']".format(sfn) 276 | else: 277 | logger.critical('Failed to parse YAML in template {0}\n{1}'.format(sfn, e)) 278 | continue 279 | 280 | # Validate operators 281 | if not res_yaml: 282 | if teamcity: 283 | print "##teamcity[testFinished name='{0}']".format(sfn) 284 | continue 285 | 286 | for key in res_yaml.copy(): 287 | skey = key.rsplit(self.delimiter, 1) 288 | rkey = None 289 | operator = None 290 | if len(skey) > 1 and key.rfind('()') > 0: 291 | rkey = skey[0].rstrip(self.delimiter) 292 | operator = skey[1] 293 | 294 | if operator == 'merge()' or operator == 'imerge()' or operator == 'immutable()': 295 | res_yaml[rkey] = res_yaml[key] 296 | del res_yaml[key] 297 | elif operator == 'unset()' or operator == 'iunset()': 298 | del res_yaml[key] 299 | elif operator is not None: 300 | success = False 301 | if teamcity: 302 | print "##teamcity[testFailed name='{0}' message='Unsupported operator {1}']".format(sfn, operator) 303 | else: 304 | logger.error('Unsupported operator {0} in template {1}'.format(operator, sfn)) 305 | 306 | if show: 307 | print '### Template: {0} ###\n'.format(sfn) 308 | print yaml.safe_dump(res_yaml, default_flow_style=False) 309 | 310 | val = cerberus.Validator() 311 | try: 312 | status = val.validate(res_yaml, schema) 313 | if not status: 314 | success = False 315 | for ekey, error in val.errors.items(): 316 | if teamcity: 317 | print "##teamcity[testFailed name='{0}' message='{1}: {2}']".format(sfn, ekey, error) 318 | else: 319 | logger.error('Incorrect key {0} in template {1}: {2}'.format(ekey, sfn, error)) 320 | except Exception, e: 321 | success = False 322 | if teamcity: 323 | print "##teamcity[testFailed name='{0}' message='Failed to validate output for template: {1}']".format(sfn, e) 324 | else: 325 | logger.error('Failed to validate output for template {0}: {1}'.format(sfn, e)) 326 | 327 | if teamcity: 328 | print "##teamcity[testFinished name='{0}']".format(sfn) 329 | 330 | if teamcity: 331 | print "##teamcity[testSuiteFinished name='pepa']" 332 | 333 | return success 334 | --------------------------------------------------------------------------------