├── docs ├── changelog.md ├── contributing.md ├── images │ └── cropped-plus3it-logo-cmyk.png ├── customization │ ├── RemoveFormulaFromTop-Simple-RedHat.txt │ ├── AddFormulaToTop-Simple-RedHat.txt │ ├── AddFormulaToTop-Jinja-RedHat.txt │ ├── FormulaTop-ALL.txt │ ├── FormulaTop-ALL_reordered.txt │ ├── Example-config.yaml │ ├── index.md │ ├── SiteFormulae.md │ ├── NewFormulas.md │ └── SiteParameters.md ├── usage.d │ ├── userData-Linux-bash.sh │ ├── userData-Linux-cloud_config.yml │ └── userData-Windows.ps1 ├── troubleshooting │ ├── NoBoto3-LogSnippet.txt │ ├── Linux │ │ ├── cloud-init-output.log.md │ │ ├── index.md │ │ ├── var-log-messages.md │ │ ├── watchmaker.log.md │ │ ├── salt_call.debug.log.md │ │ ├── journald.rst │ │ └── cloud-init.log.md │ ├── index.md │ └── Windows │ │ ├── c_amazon_EC2Launch_Log_UserdataExecution.log.md │ │ ├── c_amazon_EC2Launch_v2_Log_UserdataExecution.log.md │ │ ├── c_watchmaker_logs_salt_call.debug.md │ │ ├── index.md │ │ └── c_watchmaker_logs_watchmaker.log.md ├── _static │ └── theme_overrides.css ├── gotchas │ ├── index.md │ ├── EL8-OpenSSHkeyLogins.md │ └── EL8-X11tunneling.md ├── scap.md ├── findings │ └── index.md ├── api.md ├── CentOS-Stream.md ├── index.md └── files │ └── bootstrap │ └── watchmaker-bootstrap.ps1 ├── Dockerfile ├── requirements ├── docs-check.txt ├── check.txt ├── build.txt ├── docs.txt ├── test.txt └── basics.txt ├── src └── watchmaker │ ├── utils │ ├── imds │ │ ├── __init__.py │ │ └── detect │ │ │ ├── providers │ │ │ ├── __init__.py │ │ │ ├── provider.py │ │ │ ├── azure_provider.py │ │ │ └── aws_provider.py │ │ │ └── __init__.py │ └── urllib_utils │ │ ├── __init__.py │ │ └── request_handlers.py │ ├── managers │ ├── __init__.py │ └── worker_manager.py │ ├── workers │ ├── __init__.py │ ├── base.py │ └── yum.py │ ├── status │ ├── providers │ │ ├── __init__.py │ │ ├── abstract.py │ │ ├── azure.py │ │ └── aws.py │ └── __init__.py │ ├── static │ ├── __init__.py │ └── config.yaml │ ├── __main__.py │ ├── conditions │ └── __init__.py │ ├── exceptions │ └── __init__.py │ ├── cli.py │ └── config │ ├── status │ └── __init__.py │ └── __init__.py ├── AUTHORS.md ├── Makefile ├── .bumpversion.cfg ├── tests ├── conftest.py ├── imds-providers │ ├── test_azure_provider.py │ ├── test_imds.py │ └── test_aws_provider.py ├── resources │ ├── config_without_status.yaml │ ├── config_with_status.yaml │ ├── config_with_computer_name_pattern.yaml │ └── config_with_computer_name_and_pattern.yaml ├── status │ └── test_status.py ├── test_config.py ├── test_watchmaker.py └── test_status_config.py ├── .gitlab-ci.yml ├── .readthedocs.yaml ├── .github ├── workflows │ ├── lint.yml │ ├── dependabot_hack.yml │ ├── codeql-analysis.yml │ └── integration.yml └── dependabot.yml ├── ci ├── local │ ├── Dockerfile │ └── build_dox.sh ├── pyinstaller │ ├── win │ │ └── hook-watchmaker.py │ └── elx │ │ └── hook-watchmaker.py ├── Dockerfile ├── Dockerfile.pyapp ├── prep_docker.sh ├── build.sh ├── build.ps1 ├── prep_docker_pyapp.sh └── build_pyapp.sh ├── .gitattributes ├── .editorconfig ├── README.md ├── .gitignore ├── .mergify.yml ├── .gitmodules └── pyproject.toml /docs/changelog.md: -------------------------------------------------------------------------------- 1 | ../CHANGELOG.md -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | ../CONTRIBUTING.md -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM plus3it/tardigrade-ci:0.28.4 2 | -------------------------------------------------------------------------------- /requirements/docs-check.txt: -------------------------------------------------------------------------------- 1 | pygments==2.19.2 2 | -------------------------------------------------------------------------------- /requirements/check.txt: -------------------------------------------------------------------------------- 1 | ruff==0.14.9 2 | twine==6.2.0 3 | -------------------------------------------------------------------------------- /src/watchmaker/utils/imds/__init__.py: -------------------------------------------------------------------------------- 1 | """IMDS module.""" 2 | -------------------------------------------------------------------------------- /requirements/build.txt: -------------------------------------------------------------------------------- 1 | boto3==1.42.9 2 | pyinstaller==6.17.0 3 | -------------------------------------------------------------------------------- /src/watchmaker/managers/__init__.py: -------------------------------------------------------------------------------- 1 | """Watchmaker managers module.""" 2 | -------------------------------------------------------------------------------- /src/watchmaker/workers/__init__.py: -------------------------------------------------------------------------------- 1 | """Watchmaker workers module.""" 2 | -------------------------------------------------------------------------------- /src/watchmaker/status/providers/__init__.py: -------------------------------------------------------------------------------- 1 | """Status Providers Module.""" 2 | -------------------------------------------------------------------------------- /src/watchmaker/utils/imds/detect/providers/__init__.py: -------------------------------------------------------------------------------- 1 | """Providers module.""" 2 | -------------------------------------------------------------------------------- /src/watchmaker/static/__init__.py: -------------------------------------------------------------------------------- 1 | """Loads static module for path operations.""" 2 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Authors 2 | 3 | * Plus3IT Maintainers of Watchmaker - projects@plus3it.com 4 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | -r docs-check.txt 2 | 3 | myst-parser==4.0.1 4 | sphinx==8.2.3 5 | sphinx-rtd-theme==3.0.2 6 | -------------------------------------------------------------------------------- /docs/images/cropped-plus3it-logo-cmyk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/plus3it/watchmaker/HEAD/docs/images/cropped-plus3it-logo-cmyk.png -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include $(shell test -f .tardigrade-ci || curl -sSL -o .tardigrade-ci "https://raw.githubusercontent.com/plus3it/tardigrade-ci/master/bootstrap/Makefile.bootstrap"; echo .tardigrade-ci) 2 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.29.4 3 | commit = False 4 | tag = False 5 | tag_name = {new_version} 6 | 7 | [bumpversion:file:pyproject.toml] 8 | 9 | [bumpversion:file:docs/conf.py] 10 | -------------------------------------------------------------------------------- /docs/customization/RemoveFormulaFromTop-Simple-RedHat.txt: -------------------------------------------------------------------------------- 1 | 'G@os_family:RedHat': 2 | - name-computer 3 | - scap.content 4 | - ash-linux.vendor 5 | - ash-linux.stig 6 | - ash-linux.iavm 7 | ## - scap.scan 8 | -------------------------------------------------------------------------------- /docs/customization/AddFormulaToTop-Simple-RedHat.txt: -------------------------------------------------------------------------------- 1 | 'G@os_family:RedHat': 2 | - name-computer 3 | - scap.content 4 | - ash-linux.vendor 5 | - ash-linux.stig 6 | - ash-linux.iavm 7 | - cribl-agent 8 | - scap.scan 9 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """PyTest configuration.""" 2 | 3 | import sys 4 | 5 | 6 | def pytest_configure(config): 7 | """Set system to recognize that its in a test environment.""" 8 | sys._called_from_test = True 9 | 10 | 11 | def pytest_unconfigure(config): 12 | """Unset test environment.""" 13 | del sys._called_from_test 14 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: ${IMAGE} 2 | 3 | pages: 4 | stage: deploy 5 | before_script: 6 | - pip3 install --upgrade -r requirements/docs.txt 7 | - pip3 install -e . 8 | script: 9 | - sphinx-build -a -E -b html docs public 10 | artifacts: 11 | paths: 12 | - public 13 | only: 14 | - main 15 | tags: 16 | - pages 17 | -------------------------------------------------------------------------------- /docs/customization/AddFormulaToTop-Jinja-RedHat.txt: -------------------------------------------------------------------------------- 1 | 'G@os_family:RedHat': 2 | - name-computer 3 | - scap.content 4 | - ash-linux.vendor 5 | - ash-linux.stig 6 | - ash-linux.iavm 7 | {%- if salt.grains.get('watchmaker:enterprise_environment') | lower in environments %} 8 | - cribl-agent 9 | {%- endif %} 10 | - scap.scan 11 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | mock==5.2.0;python_version>="3.8" 2 | pytest==8.3.5;python_version<="3.8" and python_version>"3.7" 3 | pytest==8.4.2;python_version>="3.9" 4 | pytest-cov==5.0.0;python_version<="3.8" and python_version>"3.7" 5 | pytest-cov==7.0.0;python_version>="3.9" 6 | pytest-mock==3.14.1;python_version<="3.8" and python_version>"3.7" 7 | pytest-mock==3.15.1;python_version>="3.9" 8 | -------------------------------------------------------------------------------- /requirements/basics.txt: -------------------------------------------------------------------------------- 1 | ## Minimal tools related to building this project 2 | # CI uses the 'uv' CLI as the front-end build tool (installed via pip here). 3 | # The PEP 517 backend is defined in pyproject.toml [build-system] and will be 4 | # resolved automatically by the frontend. We also pin it here for reproducibility 5 | # in environments that choose to invoke the backend directly. 6 | uv==0.9.17 7 | -------------------------------------------------------------------------------- /src/watchmaker/utils/urllib_utils/__init__.py: -------------------------------------------------------------------------------- 1 | """Exposes urllib imports with additional request handlers.""" 2 | 3 | from urllib import error, parse, request # noqa: F401 4 | 5 | from watchmaker.conditions import HAS_BOTO3 6 | 7 | if HAS_BOTO3: 8 | from watchmaker.utils.urllib_utils.request_handlers import S3Handler 9 | 10 | request.install_opener(request.build_opener(S3Handler)) 11 | -------------------------------------------------------------------------------- /docs/usage.d/userData-Linux-bash.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | PYPI_URL=https://pypi.org/simple 3 | 4 | # Install pip 5 | python3 -m ensurepip 6 | 7 | # Install setup dependencies 8 | python3 -m pip install --index-url="$PYPI_URL" uv 9 | 10 | # Install Watchmaker 11 | uv pip install --index-url="$PYPI_URL" --upgrade watchmaker 12 | 13 | # Run Watchmaker 14 | watchmaker --log-level debug --log-dir=/var/log/watchmaker 15 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: "ubuntu-22.04" 5 | tools: 6 | python: "3.12" 7 | 8 | formats: all 9 | 10 | sphinx: 11 | builder: html 12 | configuration: docs/conf.py 13 | fail_on_warning: true 14 | 15 | python: 16 | install: 17 | - requirements: requirements/docs.txt 18 | - method: pip 19 | path: . 20 | 21 | submodules: 22 | include: all 23 | recursive: true 24 | -------------------------------------------------------------------------------- /src/watchmaker/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Entrypoint module, in case you use `python -m watchmaker`. 3 | 4 | Why does this file exist, and why __main__? For more info, read: 5 | 6 | - https://www.python.org/dev/peps/pep-0338/ 7 | - https://docs.python.org/2/using/cmdline.html#cmdoption-m 8 | - https://docs.python.org/3/using/cmdline.html#cmdoption-m 9 | """ 10 | 11 | from watchmaker.cli import main 12 | 13 | if __name__ == "__main__": 14 | main() 15 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Run lint and static analyis checks 2 | on: 3 | pull_request: 4 | 5 | concurrency: 6 | group: lint-${{ github.head_ref || github.ref }} 7 | cancel-in-progress: true 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | lint: 14 | uses: plus3it/actions-workflows/.github/workflows/lint.yml@269d875599c92395f7fa99cab43edc1820798e61 15 | with: 16 | tardigradelint-target: -o python/lint lint 17 | -------------------------------------------------------------------------------- /docs/usage.d/userData-Linux-cloud_config.yml: -------------------------------------------------------------------------------- 1 | # cloud-config 2 | 3 | runcmd: 4 | - | 5 | PYPI_URL=https://pypi.org/simple 6 | 7 | # Install pip 8 | python3 -m ensurepip 9 | 10 | # Install setup dependencies 11 | python3 -m pip install --index-url="$PYPI_URL" uv 12 | 13 | # Install Watchmaker 14 | uv pip install --index-url="$PYPI_URL" --upgrade watchmaker 15 | 16 | # Run Watchmaker 17 | watchmaker --log-level debug --log-dir=/var/log/watchmaker 18 | -------------------------------------------------------------------------------- /ci/local/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/docker/library/python:3.12-alpine 2 | 3 | COPY requirements/basics.txt requirements/basics.txt 4 | COPY requirements/docs.txt requirements/docs.txt 5 | COPY requirements/docs-check.txt requirements/docs-check.txt 6 | COPY ci/local/build_dox.sh /bin 7 | 8 | RUN pip install -r requirements/basics.txt 9 | RUN uv pip install --system -r requirements/docs.txt && \ 10 | uv pip install --system -r requirements/docs-check.txt 11 | 12 | CMD build_dox.sh 13 | -------------------------------------------------------------------------------- /docs/customization/FormulaTop-ALL.txt: -------------------------------------------------------------------------------- 1 | {%- set environments = ['dev', 'test', 'prod', 'dx'] %} 2 | 3 | base: 4 | 'G@os_family:RedHat': 5 | - name-computer 6 | - scap.content 7 | - ash-linux.vendor 8 | - ash-linux.stig 9 | - ash-linux.iavm 10 | - scap.scan 11 | 12 | 'G@os_family:Windows': 13 | - name-computer 14 | - pshelp 15 | - netbanner.custom 16 | - ash-windows.stig 17 | - ash-windows.iavm 18 | - ash-windows.delta 19 | - scap.scan 20 | - ash-windows.custom 21 | -------------------------------------------------------------------------------- /docs/customization/FormulaTop-ALL_reordered.txt: -------------------------------------------------------------------------------- 1 | {%- set environments = ['dev', 'test', 'prod', 'dx'] %} 2 | 3 | base: 4 | 'G@os_family:RedHat': 5 | - name-computer 6 | - ash-linux.vendor 7 | - ash-linux.stig 8 | - ash-linux.iavm 9 | - scap.content 10 | - scap.scan 11 | 12 | 'G@os_family:Windows': 13 | - name-computer 14 | - pshelp 15 | - netbanner.custom 16 | - ash-windows.stig 17 | - ash-windows.iavm 18 | - ash-windows.delta 19 | - ash-windows.custom 20 | - scap.scan 21 | -------------------------------------------------------------------------------- /docs/troubleshooting/NoBoto3-LogSnippet.txt: -------------------------------------------------------------------------------- 1 | 2023-06-22 14:26:59,192 [backoff][INFO ][4908]: Backing off urlopen_retry(...) for 0.6s (urllib.error.URLError: ) 2 | 2023-06-22 14:26:59,803 [backoff][ERROR][4908]: Giving up urlopen_retry(...) after 5 tries (urllib.error.URLError: ) 3 | 2023-06-22 14:26:59,803 [watchmaker.config][CRITICAL][4908]: Could not read config file from the provided value "s3:////config.yaml"! Check that the config is available. 4 | -------------------------------------------------------------------------------- /docs/_static/theme_overrides.css: -------------------------------------------------------------------------------- 1 | /* override nav and table width restrictions */ 2 | @media screen and (min-width: 767px) { 3 | .wy-nav-content { 4 | max-width: 50% !important; 5 | } 6 | 7 | .wy-table-responsive table td { 8 | /* !important prevents the common CSS stylesheets from 9 | overriding this as on RTD they are loaded after this stylesheet */ 10 | white-space: normal !important; 11 | } 12 | 13 | .wy-table-responsive { 14 | overflow: visible !important; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /docs/troubleshooting/Linux/cloud-init-output.log.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # The `/var/log/cloud-init-output.log` Log-File 11 | 12 | This is the default location where the Red Hat packaged version of the `cloud-init` service for Enterprise Linux 8 and higher writes its _summary_ log-output to. Primary content of potential troubleshooting-interest that can get logged here is the output from userData scripts. 13 | -------------------------------------------------------------------------------- /ci/pyinstaller/win/hook-watchmaker.py: -------------------------------------------------------------------------------- 1 | """Pyinstaller hook for watchmaker standalone.""" 2 | 3 | from PyInstaller.utils.hooks import ( 4 | collect_data_files, 5 | collect_dynamic_libs, 6 | collect_submodules, 7 | copy_metadata, 8 | ) 9 | 10 | datas = [ 11 | ("src/watchmaker/static", "watchmaker/static"), 12 | ] 13 | binaries = [] 14 | hiddenimports = [ 15 | "boto3", 16 | ] 17 | datas += copy_metadata("watchmaker", recursive=True) 18 | datas += collect_data_files("watchmaker") 19 | binaries += collect_dynamic_libs("watchmaker") 20 | hiddenimports += collect_submodules("watchmaker") 21 | -------------------------------------------------------------------------------- /src/watchmaker/conditions/__init__.py: -------------------------------------------------------------------------------- 1 | """Conditions module.""" 2 | 3 | HAS_BOTO3 = False 4 | try: 5 | import boto3 # noqa: F401 6 | 7 | HAS_BOTO3 = True 8 | except ImportError: 9 | pass 10 | 11 | HAS_AZURE = False 12 | try: 13 | from azure.core import pipeline # noqa: F401 14 | from azure.identity import ( # noqa: F401 15 | AzureCliCredential, 16 | _credentials, 17 | ) 18 | from azure.mgmt.resource import ( # noqa: F401 19 | ResourceManagementClient, 20 | resources, 21 | ) 22 | 23 | HAS_AZURE = True 24 | except ImportError: 25 | pass 26 | -------------------------------------------------------------------------------- /tests/imds-providers/test_azure_provider.py: -------------------------------------------------------------------------------- 1 | """Providers main test module.""" 2 | 3 | from unittest.mock import patch 4 | 5 | from watchmaker.utils.imds.detect.providers.azure_provider import AzureProvider 6 | 7 | 8 | @patch.object(AzureProvider, "_AzureProvider__is_valid_server") 9 | def test_metadata_server_check(provider_mock): 10 | """Tests metadata server check.""" 11 | provider = AzureProvider() 12 | 13 | provider_mock.return_value = True 14 | 15 | assert provider.check_metadata_server() is True 16 | 17 | provider_mock.return_value = False 18 | 19 | assert provider.check_metadata_server() is False 20 | -------------------------------------------------------------------------------- /src/watchmaker/utils/imds/detect/providers/provider.py: -------------------------------------------------------------------------------- 1 | """Abstract Provider.""" 2 | 3 | import abc 4 | 5 | 6 | class AbstractProvider(abc.ABC): 7 | """ 8 | Abstract class representing a cloud provider. 9 | 10 | All concrete cloud providers should implement this. 11 | """ 12 | 13 | DEFAULT_TIMEOUT = 5 14 | identifier = "unknown" 15 | 16 | @abc.abstractmethod 17 | def identify(self): 18 | """Identify provider type.""" 19 | # pragma: no cover 20 | 21 | @abc.abstractmethod 22 | def check_metadata_server(self): 23 | """Identify via metadata server.""" 24 | # pragma: no cover 25 | -------------------------------------------------------------------------------- /docs/troubleshooting/index.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # Troubleshooting Guidance 11 | 12 | Troubleshooting Watchmaker activities can be done by checking various system logs. Logfile locations vary by OS and _may_ vary by OS-version and cloud-provider. The per-OS, logfile discussions assume that you have executed Watchmaker per the relevant OSes' direct-usage guidance: 13 | 14 | ```{toctree} 15 | :maxdepth: 1 16 | Linux/index.md 17 | Windows/index.md 18 | ``` 19 | -------------------------------------------------------------------------------- /docs/gotchas/index.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # Hardening "Gotchas" 11 | 12 | The hardening-content shipped with watchmaker includes some content that may result in degredation of the hardened-systems' user experience. We will try to document, here, those gotchas that we discover or that we are able to verify from user-submitted issue-reports. 13 | 14 | ```{toctree} 15 | :maxdepth: 1 16 | EL7-sudo.md 17 | EL8-X11tunneling.md 18 | EL8-OpenSSHkeyLogins.md 19 | ``` 20 | 21 | -------------------------------------------------------------------------------- /src/watchmaker/status/providers/abstract.py: -------------------------------------------------------------------------------- 1 | """Abstract Status Provider.""" 2 | 3 | import abc 4 | 5 | 6 | class AbstractStatusProvider(abc.ABC): 7 | """ 8 | Abstract class representing a watchmaker status cloud provider. 9 | 10 | All concrete watchmaker status cloud providers should implement this. 11 | """ 12 | 13 | DEFAULT_TIMEOUT = 5 14 | identifier = "unknown" 15 | 16 | @abc.abstractmethod 17 | def initialize(self): 18 | """Initialize provider.""" 19 | # pragma: no cover 20 | 21 | @abc.abstractmethod 22 | def update_status(self, key, status, required): 23 | """Identify via metadata server.""" 24 | # pragma: no cover 25 | -------------------------------------------------------------------------------- /src/watchmaker/workers/base.py: -------------------------------------------------------------------------------- 1 | """Watchmaker base worker.""" 2 | 3 | import abc 4 | import logging 5 | 6 | 7 | class WorkerBase: 8 | """Define the architecture of a Worker.""" 9 | 10 | def __init__(self, system_params, *args, **kwargs): 11 | self.log = logging.getLogger(f"{__name__}.{self.__class__.__name__}") 12 | 13 | self.system_params = system_params 14 | WorkerBase.args = args 15 | WorkerBase.kwargs = kwargs 16 | 17 | @abc.abstractmethod 18 | def before_install(self): 19 | """Add before_install method to all child classes.""" 20 | 21 | @abc.abstractmethod 22 | def install(self): 23 | """Add install method to all child classes.""" 24 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text eol=lf 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | *.sln merge=union 7 | *.csproj merge=union 8 | *.vbproj merge=union 9 | *.fsproj merge=union 10 | *.dbproj merge=union 11 | 12 | # Standard to msysgit 13 | *.doc diff=astextplain 14 | *.DOC diff=astextplain 15 | *.docx diff=astextplain 16 | *.DOCX diff=astextplain 17 | *.dot diff=astextplain 18 | *.DOT diff=astextplain 19 | *.pdf diff=astextplain 20 | *.PDF diff=astextplain 21 | *.rtf diff=astextplain 22 | *.RTF diff=astextplain 23 | 24 | # These files are binary and should be left untouched 25 | *.png binary 26 | 27 | #=============== 28 | #Personal git attribute settings 29 | #=============== 30 | -------------------------------------------------------------------------------- /ci/local/build_dox.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -eu 3 | # 4 | # A simple script to make executing HTML-generation easy 5 | # 6 | ########################################################################### 7 | 8 | # Navigate into the project content-directory 9 | cd /watchmaker || ( echo "Dir-change failed. Aborting..." ; exit 1 ) 10 | 11 | # Build watchmaker 12 | uv build 13 | 14 | # Install the just-built watchmaker python modules 15 | uv pip install --system ./dist/watchmaker-*.whl 16 | 17 | # Build the HTML files 18 | uv run sphinx-build -a -E -W --keep-going -b html docs dist/docs 19 | 20 | # Test the documentation 21 | uv run sphinx-build -b doctest docs dist/docs 22 | 23 | # Test documentation's link-refs 24 | uv run sphinx-build -b linkcheck docs dist/docs 25 | 26 | -------------------------------------------------------------------------------- /docs/scap.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 | 9 |
10 | 11 | # Supported SCAP Benchmarks 12 | 13 | ## Windows 14 | 15 | - Microsoft Windows Server STIG Benchmark (2019) 16 | - Microsoft Windows Server STIG Benchmark (2016) 17 | - Microsoft Windows Server STIG Benchmark (2012-r2) 18 | - Microsoft Windows STIG Benchmark (10) 19 | - Microsoft .NET Framework 4 Benchmark 20 | - Internet Explorer STIG Benchmark 21 | 22 | ## Linux 23 | 24 | - Red Hat Enterprise Linux STIG Benchmark (EL7 and EL8) 25 | - OpenSCAP Security Guide (EL7 and EL8) 26 | 27 |
28 | 29 | > New benchmark versions are incorporated as they are released 30 | -------------------------------------------------------------------------------- /ci/pyinstaller/elx/hook-watchmaker.py: -------------------------------------------------------------------------------- 1 | """Pyinstaller hook for watchmaker standalone.""" 2 | 3 | from PyInstaller.utils.hooks import ( 4 | collect_data_files, 5 | collect_dynamic_libs, 6 | collect_submodules, 7 | copy_metadata, 8 | ) 9 | 10 | datas = [ 11 | ("src/watchmaker/static", "watchmaker/static"), 12 | ("/usr/lib64/.libcrypto.so.*.hmac", "."), 13 | ("/usr/lib64/.libssl.so.*.hmac", "."), 14 | ] 15 | 16 | binaries = [ 17 | ("/usr/lib64/libcrypto.so.*", "."), 18 | ("/usr/lib64/libssl.so.*", "."), 19 | ] 20 | 21 | hiddenimports = [ 22 | "boto3", 23 | ] 24 | 25 | datas += copy_metadata("watchmaker", recursive=True) 26 | datas += collect_data_files("watchmaker") 27 | binaries += collect_dynamic_libs("watchmaker") 28 | hiddenimports += collect_submodules("watchmaker") 29 | -------------------------------------------------------------------------------- /docs/usage.d/userData-Windows.ps1: -------------------------------------------------------------------------------- 1 | 2 | $BootstrapUrl = "https://watchmaker.cloudarmor.io/releases/latest/watchmaker-bootstrap.ps1" 3 | $PythonUrl = "https://www.python.org/ftp/python/3.10.11/python-3.10.11-amd64.exe" 4 | $PypiUrl = "https://pypi.org/simple" 5 | 6 | # Use TLS 1.2+ 7 | [Net.ServicePointManager]::SecurityProtocol = "Tls12, Tls13" 8 | 9 | # Download bootstrap file 10 | $BootstrapFile = "${Env:Temp}\$(${BootstrapUrl}.split('/')[-1])" 11 | (New-Object System.Net.WebClient).DownloadFile("$BootstrapUrl", "$BootstrapFile") 12 | 13 | # Install python 14 | & "$BootstrapFile" -PythonUrl "$PythonUrl" -Verbose -ErrorAction Stop 15 | 16 | # Install Watchmaker 17 | python -m pip install --index-url="$PypiUrl" uv 18 | uv pip install --index-url="$PypiUrl" --upgrade watchmaker 19 | 20 | # Run Watchmaker 21 | watchmaker --log-level debug --log-dir=C:\Watchmaker\Logs 22 | 23 | -------------------------------------------------------------------------------- /ci/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/docker/library/almalinux:8@sha256:2d4abdee2caecd851d2d6591dfb2205ba18549bc080ad5377875c990331e41c4 2 | 3 | ARG USER=wam-builder 4 | ARG USER_UID 5 | ARG USER_GID 6 | 7 | ENV PYTHON=python3.12 8 | 9 | USER root 10 | 11 | RUN if [[ ${USER_UID} -eq 0 ]] ; then adduser ${USER} ; \ 12 | else if ! getent group "$USER_GID" ; then groupadd --gid ${USER_GID} ${USER} ; \ 13 | else GROUP_NAME=$(getent group $USER_GID | awk -F':' '{print $1}') ; groupmod -n ${USER} "$GROUP_NAME" ; fi \ 14 | && adduser --uid ${USER_UID} --gid ${USER_GID} ${USER} ; fi 15 | 16 | COPY --chown=${USER}:${USER} requirements/basics.txt /requirements/basics.txt 17 | 18 | RUN dnf install -y ${PYTHON} 19 | 20 | RUN ${PYTHON} -m ensurepip --upgrade --default-pip \ 21 | && ${PYTHON} -m pip install -r /requirements/basics.txt \ 22 | && ${PYTHON} -m pip list 23 | 24 | USER ${USER} 25 | 26 | ENV HOME="/home/${USER}" 27 | -------------------------------------------------------------------------------- /docs/findings/index.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # Common Scan Findings 11 | 12 | There is frequently more than one way to achieve a given hardening-recommendation. 13 | As such, generic security scanners may produce alerts/findings that are at odds 14 | with the actual system state implemented by Watchmaker. The following are frequently-cited 15 | findings and explanations for why a scanner may alert on the Watchmaker-managed 16 | configuration-state. 17 | 18 | ## Common Scan Findings for EL7 19 | 20 | ```{toctree} 21 | :maxdepth: 1 22 | 23 | el7.md 24 | ``` 25 | 26 | ## Common Scan Findings for EL8 27 | 28 | ```{toctree} 29 | :maxdepth: 1 30 | 31 | el8.md 32 | ``` 33 | 34 | ## Common Scan Findings for EL9 35 | 36 | ```{toctree} 37 | :maxdepth: 1 38 | 39 | el9.md 40 | ``` 41 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # see http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | end_of_line = lf 6 | trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | indent_style = space 9 | indent_size = 2 10 | charset = utf-8 11 | tab_width = 4 12 | 13 | [.bumpversion.cfg] 14 | indent_style = tab 15 | 16 | [.gitmodules] 17 | indent_style = tab 18 | 19 | [*.md] 20 | trim_trailing_whitespace = false 21 | 22 | [*.rst] 23 | trim_trailing_whitespace = false 24 | 25 | [*.txt] 26 | trim_trailing_whitespace = false 27 | indent_size = none 28 | 29 | [*.py] 30 | indent_size = 4 31 | 32 | [Makefile] 33 | indent_style = tab 34 | indent_size = 1 35 | 36 | [Makefile.*] 37 | indent_style = tab 38 | indent_size = 1 39 | 40 | [LICENSE] 41 | indent_size = none 42 | 43 | [src/watchmaker/static/salt/formulas/**] 44 | indent_size = none 45 | insert_final_newline = none 46 | trim_trailing_whitespace = false 47 | 48 | [src/watchmaker/static/salt/content/**] 49 | indent_size = none 50 | insert_final_newline = none 51 | trim_trailing_whitespace = false 52 | -------------------------------------------------------------------------------- /ci/Dockerfile.pyapp: -------------------------------------------------------------------------------- 1 | FROM rust:1.91.1-slim AS rust 2 | 3 | FROM public.ecr.aws/docker/library/almalinux:8@sha256:2d4abdee2caecd851d2d6591dfb2205ba18549bc080ad5377875c990331e41c4 4 | 5 | ARG USER=wam-builder 6 | ARG USER_UID 7 | ARG USER_GID 8 | ARG UV_VERSION 9 | 10 | USER root 11 | 12 | RUN if [ ${USER_UID} -eq 0 ] ; then adduser ${USER} ; \ 13 | else if ! getent group "$USER_GID" ; then groupadd --gid ${USER_GID} ${USER} ; \ 14 | else GROUP_NAME=$(getent group $USER_GID | awk -F':' '{print $1}') ; groupmod -n ${USER} "$GROUP_NAME" ; fi \ 15 | && adduser --uid ${USER_UID} --gid ${USER_GID} ${USER} ; fi 16 | 17 | RUN dnf install -y gcc make curl zstd 18 | 19 | USER ${USER} 20 | 21 | ENV HOME="/home/${USER}" 22 | ENV RUSTUP_HOME="${HOME}/.rustup" 23 | ENV CARGO_HOME="${HOME}/.cargo" 24 | ENV PATH="${HOME}/.local/bin:${CARGO_HOME}/bin:${PATH}" 25 | 26 | # Copy Rust toolchain from rust stage 27 | COPY --from=rust --chown=${USER}:${USER} /usr/local/cargo ${CARGO_HOME} 28 | COPY --from=rust --chown=${USER}:${USER} /usr/local/rustup ${RUSTUP_HOME} 29 | 30 | # Install uv 31 | RUN curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh 32 | -------------------------------------------------------------------------------- /docs/troubleshooting/Linux/index.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # Linux Log-Files 11 | 12 | The logfiles to pay most attention to when running Watchmaker on Enterprise Linux distros (Red Hat, CentOS, Oracle Enterprise, etc.) are as follows: 13 | 14 | ```{toctree} 15 | :maxdepth: 1 16 | watchmaker.log.md 17 | salt_call.debug.log.md 18 | var-log-messages.md 19 | cloud-init.log.md 20 | cloud-init-output.log.md 21 | ``` 22 | 23 | The above are specifed in the order most-frequently used to determine execution issues. 24 | 25 | Note that the troubleshooting discussions assume that `watchmaker` execution has been effected directly through the `cloud-init` service. If `watchmaker` is being executed by other means, the above files may have no relevance to issues encountered running `watchmaker` (the `cloud-init.log` and `cloud-init-output.log`), may not exist in the documented-locations (`salt_call.debug.log` and `watchmaker.log`) and may not even exist at all (`watchmaker.log`). 26 | -------------------------------------------------------------------------------- /.github/workflows/dependabot_hack.yml: -------------------------------------------------------------------------------- 1 | # Support creating Dependabot PRs for GitHub releases 2 | 3 | # Dependabot supports updating GitHub Actions (this file) whenever a project creates 4 | # a new tag/release. By listing the projects/tools in the action, we get a PR 5 | # from Dependabot for any release of these projects. The version is then parsed 6 | # from this file, in the `*/install` targets in the tardigrade-ci `Makefile`. 7 | 8 | # Pinning tool versions this way should be a last resort, since GitHub prevents 9 | # services like Mergify from automatically merging pull requests that modify 10 | # `.github/workflows`. Only add a tool here if there is no other option for pinning 11 | # the version. 12 | name: Dependabot hack 13 | on: 14 | push: 15 | branches: 16 | - never-trigger-this-dependabot-hack-workflow 17 | 18 | jobs: 19 | dependabot_hack: 20 | name: Track project/tool versions with Dependabot 21 | runs-on: ubuntu-latest 22 | steps: 23 | # Keep these sorted alphabetically by /, separated by an empty line 24 | 25 | - uses: astral-sh/python-build-standalone@20251031 26 | 27 | - uses: ofek/pyapp@v0.29.0 28 | -------------------------------------------------------------------------------- /docs/api.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # API Reference 11 | 12 | ## watchmaker 13 | 14 | ```{eval-rst} 15 | .. automodule:: watchmaker 16 | ``` 17 | 18 | ### watchmaker.managers 19 | 20 | ```{eval-rst} 21 | .. automodule:: watchmaker.managers 22 | ``` 23 | 24 | #### watchmaker.managers.platform_manager 25 | 26 | ```{eval-rst} 27 | .. automodule:: watchmaker.managers.platform_manager 28 | ``` 29 | 30 | #### watchmaker.managers.worker_manager 31 | 32 | ```{eval-rst} 33 | .. automodule:: watchmaker.managers.worker_manager 34 | ``` 35 | 36 | ### watchmaker.workers 37 | 38 | ```{eval-rst} 39 | .. automodule:: watchmaker.workers 40 | ``` 41 | 42 | #### watchmaker.workers.base 43 | 44 | ```{eval-rst} 45 | .. automodule:: watchmaker.workers.base 46 | ``` 47 | 48 | #### watchmaker.workers.salt 49 | 50 | ```{eval-rst} 51 | .. automodule:: watchmaker.workers.salt 52 | ``` 53 | 54 | #### watchmaker.workers.yum 55 | 56 | ```{eval-rst} 57 | .. automodule:: watchmaker.workers.yum 58 | ``` 59 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | open-pull-requests-limit: 10 8 | allow: 9 | - dependency-type: direct 10 | - dependency-type: indirect 11 | groups: 12 | python: 13 | patterns: 14 | - "*" 15 | exclude-patterns: 16 | - "pyinstaller" 17 | 18 | - package-ecosystem: gitsubmodule 19 | directory: "/" 20 | schedule: 21 | interval: weekly 22 | open-pull-requests-limit: 10 23 | groups: 24 | gitsubmodule: 25 | patterns: 26 | - "*" 27 | 28 | - package-ecosystem: docker 29 | directory: "/ci" 30 | schedule: 31 | interval: weekly 32 | open-pull-requests-limit: 10 33 | groups: 34 | docker: 35 | patterns: 36 | - "*" 37 | ignore: 38 | - dependency-name: "docker/library/almalinux" 39 | update-types: ["version-update:semver-major"] 40 | 41 | - package-ecosystem: github-actions 42 | directory: / 43 | schedule: 44 | interval: weekly 45 | groups: 46 | github-actions: 47 | patterns: 48 | - "*" 49 | 50 | - package-ecosystem: docker 51 | directory: / 52 | schedule: 53 | interval: weekly 54 | groups: 55 | docker: 56 | patterns: 57 | - "*" 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License](https://img.shields.io/github/license/plus3it/watchmaker.svg)](./LICENSE) 2 | [![Build and Test Status](https://github.com/plus3it/watchmaker/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/plus3it/watchmaker/actions/workflows/build.yml) 3 | [![Release and Publish Status](https://github.com/plus3it/watchmaker/actions/workflows/release.yml/badge.svg?branch=main)](https://github.com/plus3it/watchmaker/actions/workflows/release.yml) 4 | [![Latest Version](https://img.shields.io/pypi/v/watchmaker.svg?label=version)](https://pypi.python.org/pypi/watchmaker) 5 | 6 | # Watchmaker 7 | 8 | Applied Configuration Management 9 | 10 | ## Overview 11 | 12 | Watchmaker is a Python package that helps bootstrap a vanilla OS image and 13 | apply an OS configuration. Watchmaker itself reads a simple YAML configuration 14 | file, which can be hosted on the local filesystem or on a web server. 15 | 16 | Complex configuration management (CM) environments may be layered in as part of 17 | the provisioning framework. Watchmaker includes a default configuration that 18 | will install Salt and a handful Salt Formulas that can be used to harden a 19 | system to DISA STIG standards, as well as integrate with common enterprise 20 | services. 21 | 22 | ## Documentation 23 | 24 | For more information on installing and using Watchmaker, go to 25 | . 26 | 27 | ## Changelog 28 | 29 | Changelog can be found at . 30 | -------------------------------------------------------------------------------- /docs/CentOS-Stream.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # CentOS Stream Discontinuation Notes 11 | 12 | With the CentOS maintainers having discontinued CentOS Stream 8 at the end of 13 | May of 2024, access to security-update and feature content is no longer 14 | available within the repositories activated by default in templates and systems 15 | deployed prior to that date. As a result, hardening-operations that require the 16 | installation of either additional packages or updates to already-installed 17 | packages will fail. This may be worked around by deactivating the standard 18 | repositories and creating "vault" repositories from them. This may be done with 19 | a quick script like: 20 | 21 | ``` 22 | # ( 23 | cd /etc/yum.repos.d && 24 | for RepoFile in CentOS-Stream-{BaseOS,AppStream,Extras{,-common},HighAvailability,NFV,PowerTools,RealTime,ResilientStorage}.repo 25 | do 26 | sed -e '/^mirrorlist/s/^/##/' \ 27 | -e '/baseurl=/s/^#*//' \ 28 | -e '/baseurl=/s/mirror\.centos\.org/vault.centos.org/' \ 29 | -e '/\[/s/^\[/&vault-/' \ 30 | -e '/^name/s/$/ (Vault)/' \ 31 | "${RepoFile}" > "${RepoFile//.repo/-Vault.repo}" 32 | done 33 | ) 34 | # dnf config-manager --save --set-disabled appstream baseos extras 35 | # dnf config-manager --save --set-enabled vault-{appstream,baseos,extras} 36 | ``` 37 | -------------------------------------------------------------------------------- /src/watchmaker/utils/imds/detect/providers/azure_provider.py: -------------------------------------------------------------------------------- 1 | """Azure Provider.""" 2 | 3 | import logging 4 | from urllib import error as urllib_error 5 | 6 | from watchmaker import utils 7 | from watchmaker.utils.imds.detect.providers.provider import AbstractProvider 8 | 9 | 10 | class AzureProvider(AbstractProvider): 11 | """Concrete implementation of the Azure cloud provider.""" 12 | 13 | identifier = "azure" 14 | 15 | def __init__(self, logger=None): 16 | self.logger = logger or logging.getLogger(__name__) 17 | self.metadata_url = ( 18 | "http://169.254.169.254/metadata/instance?api-version=2021-02-01" 19 | ) 20 | self.headers = {"Metadata": "true"} 21 | 22 | def identify(self): 23 | """Identify Azure using all the implemented options.""" 24 | self.logger.info("Try to identify Azure") 25 | return self.check_metadata_server() 26 | 27 | def check_metadata_server(self): 28 | """Identify Azure via metadata server.""" 29 | self.logger.debug("Checking Azure metadata") 30 | try: 31 | return self.__is_valid_server() 32 | except urllib_error.URLError as ex: 33 | self.logger.warning("Error while checking server %s", str(ex)) 34 | return False 35 | 36 | def __is_valid_server(self): 37 | """Retrieve Azure metadata.""" 38 | response = utils.urlopen_retry(self.metadata_url, self.DEFAULT_TIMEOUT) 39 | http_ok = 200 40 | return response.status == http_ok 41 | -------------------------------------------------------------------------------- /docs/troubleshooting/Windows/c_amazon_EC2Launch_Log_UserdataExecution.log.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # The `C:\ProgramData\Amazon\EC2-Windows\Launch\Log\UserdataExecution.log` Log-File 11 | 12 | This file tracks the top-level execution of any tasks specified in a Windows Server EC2's userData payload. This file should _always_ exist. The primary reasons that it may not exist are: 13 | 14 | - The EC2 was launched from an AMI that leverages the ``EC2Launch v2`` method 15 | - The EC2 was launched from an AMI that does not have the tooling to support parsing/executing a userData 16 | 17 | Windows AMIs published through the Amazon/Microsoft partnership will always contain the tooling to support either the ``EC2Launch`` or ``EC2Launch v2`` parsing/execution of userData payloads: 18 | - Windows Server 2022 and higher AMIs use the ``EC2Launch v2`` userData payload-handler 19 | - Windows Server 2012, 2016 and 2019 AMIs use the ``EC2Launch`` userData payload-handler unless their AMI-names start with the string "``EC2LaunchV2-``" 20 | 21 | To get a list of Windows AMIs that leverage the legacy ``EC2Launch`` userData payload-handler, use a (CLI) query similar to: 22 | 23 | ``` 24 | aws ec2 describe-images \ 25 | --owner amazon \ 26 | --filters 'Name=name,Values=Windows_Server-201*' \ 27 | --query 'Images[].[CreationDate,ImageId,Name]' \ 28 | --output text 29 | ``` 30 | 31 | -------------------------------------------------------------------------------- /ci/prep_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu -o pipefail 4 | 5 | echo "***********************************************************************" 6 | echo "Prepping for Docker Watchmaker build on $(lsb_release -a)" 7 | echo "***********************************************************************" 8 | 9 | DOCKER_INSTANCE_NAME=wam-builder 10 | echo "DOCKER_INSTANCE_NAME:${DOCKER_INSTANCE_NAME}" 11 | 12 | WORKDIR="${WORKDIR:-${TRAVIS_BUILD_DIR:-}}" 13 | WORKDIR="${WORKDIR:-${PWD}}" 14 | echo "WORKDIR:${WORKDIR}" 15 | 16 | if [ -n "${WORKDIR}" ]; then 17 | 18 | echo "Building Docker container..." 19 | docker build \ 20 | --build-arg USER_UID="$(id -u)" \ 21 | --build-arg USER_GID="$(id -g)" \ 22 | -t $DOCKER_INSTANCE_NAME -f "${WORKDIR}/ci/Dockerfile" . 23 | 24 | echo "Building using container and workdir (${WORKDIR})..." 25 | docker run --detach --privileged \ 26 | --user "$(id -u):$(id -g)" \ 27 | --volume="${WORKDIR}:${WORKDIR}" \ 28 | --workdir "${WORKDIR}" \ 29 | --name "${DOCKER_INSTANCE_NAME}" \ 30 | "${DOCKER_INSTANCE_NAME}:latest" \ 31 | init 32 | 33 | echo "Building the standalone using ci/build.sh..." 34 | docker exec "${DOCKER_INSTANCE_NAME}" chmod +x ci/build.sh 35 | docker exec "${DOCKER_INSTANCE_NAME}" ci/build.sh 36 | 37 | echo "Stopping the docker container ${DOCKER_INSTANCE_NAME}..." 38 | docker stop "${DOCKER_INSTANCE_NAME}" 39 | echo "Removing the docker image ${DOCKER_INSTANCE_NAME}..." 40 | docker rm "${DOCKER_INSTANCE_NAME}" 41 | 42 | else 43 | 44 | echo "No WORKDIR provided so not building..." 45 | 46 | fi 47 | -------------------------------------------------------------------------------- /docs/troubleshooting/Windows/c_amazon_EC2Launch_v2_Log_UserdataExecution.log.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # The `C:\ProgramData\Amazon\EC2Launch\log\agent.log` Log-File 11 | 12 | ```{eval-rst} 13 | .. warning:: Watchmaker has not yet been tested with the EC2Launch v2 service. As of the writing of this document, the recommended userData content does not function under EC2Launch v2. This document is *very* beta, and primarily acts as a place-holder. 14 | ``` 15 | 16 | This file tracks the top-level execution of any tasks specified in a Windows Server EC2's userData payload. This file should _always_ exist. The primary reasons that it may not exist are: 17 | 18 | - The EC2 was launched from an AMI that leverages the (legacy) ``EC2Launch`` method 19 | - The EC2 was launched from an AMI that does not have the tooling to support parsing/executing a userData 20 | 21 | Windows AMIs published through the Amazon/Microsoft partnership will always contain the tooling to support either the ``EC2Launch`` or ``EC2Launch v2`` parsing/execution of userData payloads: 22 | - Windows Server 2022 and higher AMIs use the ``EC2Launch v2`` userData payload-handler 23 | - Windows Server 2012, 2016 and 2019 AMIs use the ``EC2Launch`` userData payload-handler unless their AMI-names start with the string "``EC2LaunchV2-``" 24 | 25 | To get a list of Windows 2012, 2016 or 2019 AMIs that leverage the ``EC2Launch v2`` userData payload-handler, use a (CLI) query similar to: 26 | 27 | ``` 28 | aws ec2 describe-images \ 29 | --owner amazon \ 30 | --filters 'Name=name,Values=EC2LaunchV2-Windows_Server-201*' \ 31 | --query 'Images[].[CreationDate,ImageId,Name]' \ 32 | --output text 33 | ``` 34 | 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | #*.cab 13 | #*.msi 14 | #*.msm 15 | #*.msp 16 | 17 | # ========================= 18 | # Operating System Files 19 | # ========================= 20 | 21 | # OSX 22 | # ========================= 23 | 24 | .DS_Store 25 | .AppleDouble 26 | .LSOverride 27 | 28 | # Icon must end with two \r 29 | Icon 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear on external disk 35 | .Spotlight-V100 36 | .Trashes 37 | 38 | # Directories potentially created on remote AFP share 39 | .AppleDB 40 | .AppleDesktop 41 | Network Trash Folder 42 | Temporary Items 43 | .apdisk 44 | 45 | #=============== 46 | #Personal git ignore settings 47 | #=============== 48 | 49 | #pycharm dirs 50 | .idea 51 | 52 | #vscode dirs 53 | .vscode/ 54 | 55 | # mkdocs documentation 56 | site/ 57 | 58 | # Packages 59 | __pycache__/ 60 | *.py[cod] 61 | *.egg 62 | *.egg-info 63 | dist 64 | build 65 | eggs 66 | .eggs 67 | parts 68 | bin 69 | var 70 | sdist 71 | wheelhouse 72 | develop-eggs 73 | .installed.cfg 74 | lib 75 | lib64 76 | venv*/ 77 | pyvenv*/ 78 | .pyinstaller/ 79 | 80 | # Installer logs 81 | pip-log.txt 82 | 83 | # Log files created by running program 84 | src/*.log 85 | tests/*.log 86 | 87 | # Directories generated by running 'tox' 88 | logfiles/ 89 | tests/logfiles/ 90 | 91 | # Environment files 92 | Pipfile 93 | Pipfile.lock 94 | 95 | # Unit test / coverage reports 96 | .pytest_cache 97 | .cache 98 | .coverage 99 | .tox 100 | .coverage.* 101 | nosetests.xml 102 | coverage.xml 103 | htmlcov 104 | 105 | # gravitybee build dir 106 | .gravitybee 107 | 108 | # tardigrade-ci 109 | .tardigrade-ci 110 | tardigrade-ci/ 111 | 112 | # PyApp build directories 113 | .pyapp/ 114 | -------------------------------------------------------------------------------- /src/watchmaker/utils/imds/detect/__init__.py: -------------------------------------------------------------------------------- 1 | """Detect module.""" 2 | 3 | import concurrent.futures 4 | import logging 5 | 6 | from watchmaker.exceptions import CloudDetectError, InvalidProviderError 7 | from watchmaker.utils.imds.detect.providers.aws_provider import AWSProvider 8 | from watchmaker.utils.imds.detect.providers.azure_provider import AzureProvider 9 | from watchmaker.utils.imds.detect.providers.provider import AbstractProvider 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | MAX_WORKERS = 10 14 | 15 | CLOUD_PROVIDERS = {"aws": AWSProvider, "azure": AzureProvider} 16 | 17 | 18 | def provider(supported_providers=None): 19 | """Identify and return identifier.""" 20 | results = [] 21 | supported_providers = supported_providers if supported_providers else [] 22 | with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: 23 | futures = [ 24 | executor.submit(identify, CLOUD_PROVIDERS[cloud_identifier]) 25 | for cloud_identifier in supported_providers 26 | if CLOUD_PROVIDERS.get(cloud_identifier) 27 | ] 28 | 29 | concurrent.futures.wait(futures) 30 | for fut in futures: 31 | try: 32 | results.append(fut.result()) 33 | except InvalidProviderError: # noqa: PERF203 34 | pass 35 | except Exception: 36 | log.exception("Unexpected exception occurred") 37 | 38 | if len(results) > 1: 39 | raise CloudDetectError 40 | 41 | if len(results) == 0: 42 | return AbstractProvider 43 | 44 | return results[0] 45 | 46 | 47 | def identify(cloud_provider): 48 | """Identify provider.""" 49 | cloud_provider_instance = cloud_provider() 50 | if cloud_provider_instance.identify(): 51 | return cloud_provider_instance 52 | 53 | log.debug("Environment is not %s", cloud_provider_instance.identifier) 54 | raise InvalidProviderError(cloud_provider_instance.identifier) 55 | -------------------------------------------------------------------------------- /.mergify.yml: -------------------------------------------------------------------------------- 1 | pull_request_rules: 2 | # For salt-formula or pyinstaller updates, trigger codebuild, merge on codebuild status 3 | - name: Trigger builds for salt-formula or pyinstaller updates 4 | conditions: 5 | - and: 6 | - author=dependabot[bot] 7 | - or: 8 | - label=submodules 9 | - title~=(?i).*pyinstaller.* 10 | actions: 11 | review: 12 | type: APPROVE 13 | message: /build 14 | 15 | - name: Merge salt-formula or pyinstaller updates 16 | conditions: 17 | - and: 18 | - author=dependabot[bot] 19 | - "#approved-reviews-by>=1" 20 | - or: 21 | - label=submodules 22 | - title~=(?i).*pyinstaller.* 23 | # Must pass codebuild jobs before merge 24 | - check-success = "test-source (rhel8)" 25 | - check-success = "test-source (rhel9)" 26 | - check-success = "test-source (win16)" 27 | - check-success = "test-source (win19)" 28 | - check-success = "test-source (win22)" 29 | - check-success = "test-standalone (rhel8)" 30 | - check-success = "test-standalone (rhel9)" 31 | - check-success = "test-standalone (win16)" 32 | - check-success = "test-standalone (win19)" 33 | - check-success = "test-standalone (win22)" 34 | actions: 35 | merge: 36 | method: merge 37 | 38 | # For regular dependabot pr, approve and merge after branch protection checks 39 | - name: Approve dependabot pull requests 40 | conditions: 41 | - author=dependabot[bot] 42 | - label!=submodules 43 | - -title~=(?i).*pyinstaller.* 44 | actions: 45 | review: 46 | type: APPROVE 47 | 48 | - name: Merge dependabot pull requests 49 | conditions: 50 | - author=dependabot[bot] 51 | - "#approved-reviews-by>=1" 52 | - label!=submodules 53 | - -title~=(?i).*pyinstaller.* 54 | actions: 55 | merge: 56 | method: merge 57 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # watchmaker 11 | 12 | Applied Configuration Management 13 | 14 | -------------- 15 | 16 | ## Overview 17 | 18 | Watchmaker is intended to help provision a system from its initial installation 19 | to its final configuration. It was inspired by a desire to eliminate static 20 | system images with embedded configuration settings (e.g. gold disks) and the 21 | pain associated with maintaining them. 22 | 23 | Watchmaker works as a sort of task runner. It consists of "_managers_" and 24 | "_workers_". A _manager_ implements common methods for multiple platforms 25 | (Linux, Windows, etc). A _worker_ exposes functionality to a user that helps 26 | bootstrap and configure the system. _Managers_ are primarily internal 27 | constructs; _workers_ expose configuration artifacts to users. Watchmaker then 28 | uses a common [configuration file](configuration) to determine what 29 | _workers_ to execute on each platform. 30 | 31 | ## Contents 32 | 33 | ```{toctree} 34 | :maxdepth: 1 35 | 36 | installation.md 37 | configuration.md 38 | usage.md 39 | customization/index.md 40 | troubleshooting/index.md 41 | gotchas/index.md 42 | faq.md 43 | scap.md 44 | findings/index.md 45 | api.md 46 | contributing.md 47 | changelog.md 48 | CentOS-Stream.md 49 | ``` 50 | 51 | ## Supported Operating Systems 52 | 53 | * Enterprise Linux 9 (RHEL/Oracle Linux/CentOS Stream/Alma Linux/Rocky Linux) 54 | * Enterprise Linux 8 (RHEL/Oracle Linux/[CentOS Stream](CentOS-Stream.md)) 55 | * Windows Server 2022 56 | * Windows Server 2019 57 | * Windows Server 2016 58 | * Windows 11 59 | * Windows 10 60 | 61 | ## Supported Python Versions 62 | 63 | * Python 3.8 and later 64 | 65 | ## Supported Salt Versions 66 | 67 | * Salt 2018.3, from 2018.3.4 and later 68 | * Salt 2019.2, from 2019.2.5 and later 69 | * Salt 300x, from 3003 and later 70 | -------------------------------------------------------------------------------- /ci/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu -o pipefail 3 | 4 | export VIRTUAL_ENV_DIR=.pyinstaller/venv 5 | 6 | VERSION=$(grep -E '^version\s*=' pyproject.toml | sed 's/^version = "\(.*\)"$/\1/') 7 | 8 | PYI_DIST_DIR=".pyinstaller/dist/${VERSION}" 9 | PYI_SPEC_DIR=".pyinstaller/spec" 10 | PYI_WORK_DIR=".pyinstaller/build" 11 | 12 | PYI_HOOK_DIR="./ci/pyinstaller/elx" 13 | PYI_SCRIPT="${PYI_SPEC_DIR}/watchmaker-standalone.py" 14 | WAM_SCRIPT="./src/watchmaker/__main__.py" 15 | WAM_FILENAME="watchmaker-${VERSION}-standalone-linux-x86_64" 16 | 17 | uv venv "$VIRTUAL_ENV_DIR" 18 | # shellcheck disable=SC1091 19 | source "${VIRTUAL_ENV_DIR}/bin/activate" 20 | 21 | echo "-----------------------------------------------------------------------" 22 | python --version 23 | uv --version 24 | echo "-----------------------------------------------------------------------" 25 | 26 | uv pip install -r requirements/build.txt 27 | uv pip install --editable . 28 | uv pip list 29 | 30 | echo "Creating standalone for watchmaker v${VERSION}..." 31 | mkdir -p "$PYI_SPEC_DIR" 32 | cp "$WAM_SCRIPT" "$PYI_SCRIPT" 33 | # Add debug argument to pyinstaller command to build standalone with debug flags 34 | # --debug all \ 35 | uv run pyinstaller --noconfirm --clean --onefile \ 36 | --name "$WAM_FILENAME" \ 37 | --runtime-tmpdir . \ 38 | --paths src \ 39 | --additional-hooks-dir "$PYI_HOOK_DIR" \ 40 | --distpath "$PYI_DIST_DIR" \ 41 | --specpath "$PYI_SPEC_DIR" \ 42 | --workpath "$PYI_WORK_DIR" \ 43 | "$PYI_SCRIPT" 44 | 45 | # Uncomment this to list the files in the standalone; can help when debugging 46 | # echo "Listing files in standalone..." 47 | # pyi-archive_viewer --log --brief --recursive "${DIST_DIR}/${WAM_FILENAME}" 48 | 49 | echo "Creating sha256 hashes of standalone binary..." 50 | (cd "$PYI_DIST_DIR"; sha256sum "$WAM_FILENAME" > "${WAM_FILENAME}.sha256") 51 | cat "${PYI_DIST_DIR}/${WAM_FILENAME}.sha256" 52 | 53 | echo "Checking standalone binary version..." 54 | eval "${PYI_DIST_DIR}/${WAM_FILENAME}" --version 55 | 56 | echo "Copying bootstrap script to dist dirs..." 57 | cp docs/files/bootstrap/watchmaker-bootstrap.ps1 "$PYI_DIST_DIR" 58 | 59 | echo "Listing files in dist dirs..." 60 | ls -alRh "$PYI_DIST_DIR"/* 61 | -------------------------------------------------------------------------------- /src/watchmaker/exceptions/__init__.py: -------------------------------------------------------------------------------- 1 | """Watchmaker exceptions module.""" 2 | 3 | 4 | class WatchmakerError(Exception): 5 | """An unknown error occurred.""" 6 | 7 | 8 | class InvalidComputerNameError(WatchmakerError): 9 | """Exception raised when computer_name does not match pattern provided.""" 10 | 11 | def __init__(self, computer_name, pattern): 12 | """Initialize with computer name and pattern.""" 13 | super().__init__( 14 | f"Computer name: {computer_name} does not match pattern {pattern}", 15 | ) 16 | 17 | 18 | class InvalidValueError(WatchmakerError): 19 | """Passed an invalid value.""" 20 | 21 | 22 | class StatusProviderError(WatchmakerError): 23 | """Status Error.""" 24 | 25 | 26 | class CloudDetectError(WatchmakerError): 27 | """Cloud Detect Error.""" 28 | 29 | 30 | class InvalidProviderError(WatchmakerError): 31 | """Invalid Provider Error.""" 32 | 33 | 34 | class OuPathRequiredError(Exception): 35 | """Exception raised when the OU path is required but not provided.""" 36 | 37 | 38 | class StatusConfigError(WatchmakerError): 39 | """Exception raised when status configuration is invalid.""" 40 | 41 | 42 | class InvalidRegexPatternError(WatchmakerError): 43 | """Exception raised when regex pattern is invalid.""" 44 | 45 | 46 | class MultiplePathsMatchError(WatchmakerError): 47 | """Exception raised when multiple paths match a glob pattern.""" 48 | 49 | 50 | class PathNotFoundError(WatchmakerError): 51 | """Exception raised when expected path is not found.""" 52 | 53 | 54 | class CloudProviderDetectionError(WatchmakerError): 55 | """Exception raised when cloud environment detection fails.""" 56 | 57 | def __init__(self, provider_identifier): 58 | """Initialize with provider identifier.""" 59 | super().__init__( 60 | f"Required Provider detected that is missing prereqs: " 61 | f"{provider_identifier}", 62 | ) 63 | 64 | 65 | class MissingURLParamError(WatchmakerError): 66 | """Exception raised when required URL parameter is missing.""" 67 | 68 | 69 | # Deprecated/renamed exceptions 70 | WatchmakerException = WatchmakerError 71 | InvalidValue = InvalidValueError 72 | OuPathRequired = OuPathRequiredError 73 | -------------------------------------------------------------------------------- /tests/resources/config_without_status.yaml: -------------------------------------------------------------------------------- 1 | watchmaker_version: ">= 0.24.0.dev" 2 | 3 | all: 4 | - salt: 5 | admin_groups: null 6 | admin_users: null 7 | computer_name: null 8 | environment: null 9 | ou_path: null 10 | salt_content: null 11 | salt_states: Highstate 12 | user_formulas: 13 | # To add extra formulas, specify them as a map of 14 | # : 15 | # The is the name of the directory in the salt file_root 16 | # where the formula will be placed. The must be a zip 17 | # file, and the zip must contain a top-level directory that, itself, 18 | # contains the actual salt formula. To "overwrite" submodule formulas, 19 | # make sure matches submodule names. E.g.: 20 | # ash-linux-formula: https://s3.amazonaws.com/salt-formulas/ash-linux-formula-master.zip 21 | # scap-formula: https://s3.amazonaws.com/salt-formulas/scap-formula-master.zip 22 | 23 | linux: 24 | - yum: 25 | repo_map: 26 | # SaltEL6: 27 | - dist: 28 | - redhat 29 | - centos 30 | el_version: 6 31 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/2019.2.8/salt-reposync-el6.repo 32 | - dist: amazon 33 | el_version: 6 34 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/2019.2.8/salt-reposync-amzn.repo 35 | # SaltEL7: 36 | - dist: 37 | - redhat 38 | - centos 39 | el_version: 7 40 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3004.2/salt-reposync-el7-python3.repo 41 | # SaltEL8: 42 | - dist: 43 | - redhat 44 | - centos 45 | el_version: 8 46 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3004.2/salt-reposync-el8-python3.repo 47 | - salt: 48 | salt_debug_log: null 49 | install_method: yum 50 | bootstrap_source: null 51 | git_repo: null 52 | salt_version: null 53 | 54 | windows: 55 | - salt: 56 | salt_debug_log: null 57 | installer_url: https://watchmaker.cloudarmor.io/repo/saltstack/salt/windows/Salt-Minion-3004.2-1-Py3-AMD64-Setup.exe 58 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [main] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [main] 20 | schedule: 21 | - cron: '36 17 * * 4' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: [python] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v6 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v4 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v4 54 | 55 | - name: Perform CodeQL Analysis 56 | uses: github/codeql-action/analyze@v4 57 | -------------------------------------------------------------------------------- /docs/troubleshooting/Linux/var-log-messages.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # The `/var/log/messages` Log-File 11 | 12 | This is Red Hat Enterprise Linux's default/primary logging location for miscellaneous system activities. Any init- or systemd-launched service that emits output to STDERR or STDOUT will typically (also) log to this file.[^1] 13 | 14 | Typically, the provisioning-administrator will wish to review this file to trace where failures in the invocation of watchmaker have failed or where errors in an instance's/VM's userData payload has encountered errors. 15 | 16 | - Search, case-insensitively, for the string "`watchmaker`" to find logged-content explicit to the execution of watchmaker. Depending how far watchmaker got before failing, there can be a significant amount of output to pore through (recommend piping to a pagination-tool such as `less`) 17 | - Search for the string "`\ cloud-init:\ `" to find logged-content related to the `cloud-init` service. This search-string will reveal execution-output made to STDOUT and STDERR by any processes initiated by the `cloud-init` service. This will _typically_ include watchmaker and any logging-enabled userData script-output. Search output will tend to be even more-significant than looking just for `watchmaker` (therefore, also recommend piping to a pagination-tool such as `less`) 18 | 19 | The use of the qualifier, "typically", in the prior bullet is required to account for different methods for invoking `watchmaker`. Some `watchmaker`-users leverage methods such as CloudFormation and other templating-engines, Ansible and other externalized provisioning-services, etc. to launch the `watchmaker` process. Those methods are outside the scope of this document. The relevant logging should be known to the user of these alternate execution-frameworks. 20 | 21 | 22 | [^1]: Some sites will explicitly disable local logging to this file. If this has been done, data that normally shows up in `/var/log/messages` _may_, instead, be found in the systemd output logs. See the [Using journald](journald.rst) document for a fuller detailing of using `journald` logging. 23 | -------------------------------------------------------------------------------- /src/watchmaker/utils/urllib_utils/request_handlers.py: -------------------------------------------------------------------------------- 1 | """Extends urllib with additional handlers.""" 2 | 3 | import io 4 | import urllib.error 5 | import urllib.request 6 | import urllib.response 7 | from email import message_from_string 8 | 9 | import boto3 10 | 11 | from watchmaker.exceptions import MissingURLParamError 12 | 13 | 14 | class BufferedIOS3Key(io.BufferedIOBase): 15 | """Add a read method to S3 key object.""" 16 | 17 | def __init__(self, key, *args, **kwargs): 18 | super().__init__(*args, **kwargs) 19 | self.read = key.get()["Body"].read 20 | 21 | 22 | class S3Handler(urllib.request.BaseHandler): 23 | """Define urllib handler for S3 objects.""" 24 | 25 | def s3_open(self, req): 26 | """Open S3 objects.""" 27 | # Credit: 28 | 29 | # The implementation was inspired mainly by the code behind 30 | # urllib.request.FileHandler.file_open(). 31 | 32 | selector = req.selector 33 | 34 | bucket_name = req.host 35 | key_name = selector[1:] 36 | 37 | if not bucket_name or not key_name: 38 | msg = "s3:///" 39 | raise MissingURLParamError(msg) 40 | 41 | try: 42 | s3_conn = self.s3_conn 43 | except AttributeError: 44 | s3_conn = self.s3_conn = boto3.resource("s3") 45 | 46 | key = s3_conn.Object(bucket_name=bucket_name, key=key_name) 47 | 48 | origurl = f"s3://{bucket_name}/{key_name}" 49 | 50 | if key is None: 51 | raise MissingURLParamError(origurl) 52 | 53 | headers = [ 54 | ("Content-type", key.content_type), 55 | ("Content-encoding", key.content_encoding), 56 | ("Content-language", key.content_language), 57 | ("Content-length", key.content_length), 58 | ("Etag", key.e_tag), 59 | ("Last-modified", key.last_modified), 60 | ] 61 | 62 | headers = message_from_string( 63 | "\n".join( 64 | f"{header}: {value}" for header, value in headers if value is not None 65 | ), 66 | ) 67 | 68 | return urllib.response.addinfourl(BufferedIOS3Key(key), headers, origurl) 69 | -------------------------------------------------------------------------------- /src/watchmaker/static/config.yaml: -------------------------------------------------------------------------------- 1 | watchmaker_version: ">= 0.27.2.dev" 2 | 3 | all: 4 | - salt: 5 | admin_groups: null 6 | admin_users: null 7 | computer_name: null 8 | environment: null 9 | ou_path: null 10 | salt_content: null 11 | salt_states: Highstate 12 | salt_version: "3007.2" 13 | user_formulas: 14 | # To add extra formulas, specify them as a map of 15 | # : 16 | # The is the name of the directory in the salt file_root 17 | # where the formula will be placed. The must be a zip 18 | # file, and the zip must contain a top-level directory that, itself, 19 | # contains the actual salt formula. To "overwrite" submodule formulas, 20 | # make sure matches submodule names. E.g.: 21 | # ash-linux-formula: https://s3.amazonaws.com/salt-formulas/ash-linux-formula-master.zip 22 | # scap-formula: https://s3.amazonaws.com/salt-formulas/scap-formula-master.zip 23 | 24 | linux: 25 | - yum: 26 | repo_map: 27 | # SaltEL8: 28 | - dist: 29 | - almalinux 30 | - centos 31 | - oracle 32 | - redhat 33 | - rocky 34 | el_version: 8 35 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3007.2/salt-reposync-onedir.repo 36 | # SaltEL9: 37 | - dist: 38 | - almalinux 39 | - centos 40 | - oracle 41 | - redhat 42 | - rocky 43 | el_version: 9 44 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3007.2/salt-reposync-onedir.repo 45 | - salt: 46 | pip_install: 47 | - dnspython 48 | salt_debug_log: null 49 | install_method: yum 50 | bootstrap_source: null 51 | git_repo: null 52 | salt_version: null 53 | 54 | windows: 55 | - salt: 56 | salt_debug_log: null 57 | installer_url: https://watchmaker.cloudarmor.io/repo/saltstack/salt/windows/Salt-Minion-3007.2-Py3-AMD64-Setup.exe 58 | 59 | status: 60 | providers: 61 | - key: "WatchmakerStatus" 62 | required: false 63 | provider_type: "aws" 64 | - key: "WatchmakerStatus" 65 | required: false 66 | provider_type: "azure" 67 | -------------------------------------------------------------------------------- /docs/customization/Example-config.yaml: -------------------------------------------------------------------------------- 1 | watchmaker_version: ">= 0.27.2.dev" 2 | 3 | all: 4 | - salt: 5 | admin_groups: null 6 | admin_users: null 7 | computer_name: null 8 | environment: null 9 | ou_path: null 10 | salt_content: null 11 | salt_states: Highstate 12 | salt_version: "3007.2" 13 | user_formulas: 14 | # To add extra formulas, specify them as a map of 15 | # : 16 | # The is the name of the directory in the salt file_root 17 | # where the formula will be placed. The must be a zip 18 | # file, and the zip must contain a top-level directory that, itself, 19 | # contains the actual salt formula. To "overwrite" submodule formulas, 20 | # make sure matches submodule names. E.g.: 21 | # ash-linux-formula: https://s3.amazonaws.com/salt-formulas/ash-linux-formula-master.zip 22 | # scap-formula: https://s3.amazonaws.com/salt-formulas/scap-formula-master.zip 23 | 24 | linux: 25 | - yum: 26 | repo_map: 27 | # SaltEL8: 28 | - dist: 29 | - almalinux 30 | - centos 31 | - oracle 32 | - redhat 33 | - rocky 34 | el_version: 8 35 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3007.2/salt-reposync-onedir.repo 36 | # SaltEL9: 37 | - dist: 38 | - almalinux 39 | - centos 40 | - oracle 41 | - redhat 42 | - rocky 43 | el_version: 9 44 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3007.2/salt-reposync-onedir.repo 45 | - salt: 46 | pip_install: 47 | - dnspython 48 | salt_debug_log: null 49 | install_method: yum 50 | bootstrap_source: null 51 | git_repo: null 52 | salt_version: null 53 | 54 | windows: 55 | - salt: 56 | salt_debug_log: null 57 | installer_url: https://watchmaker.cloudarmor.io/repo/saltstack/salt/windows/Salt-Minion-3007.2-Py3-AMD64-Setup.exe 58 | 59 | status: 60 | providers: 61 | - key: "WatchmakerStatus" 62 | required: false 63 | provider_type: "aws" 64 | - key: "WatchmakerStatus" 65 | required: false 66 | provider_type: "azure" 67 | -------------------------------------------------------------------------------- /tests/imds-providers/test_imds.py: -------------------------------------------------------------------------------- 1 | """Providers main test module.""" 2 | 3 | from unittest.mock import patch 4 | 5 | import pytest 6 | 7 | from watchmaker.utils.imds.detect import provider 8 | from watchmaker.utils.imds.detect.providers.aws_provider import AWSProvider 9 | from watchmaker.utils.imds.detect.providers.azure_provider import AzureProvider 10 | 11 | 12 | @patch.object(AWSProvider, "identify", return_value=True) 13 | @patch.object(AzureProvider, "identify", return_value=False) 14 | @patch.object( 15 | AWSProvider, 16 | "_AWSProvider__request_token", 17 | return_value=(None), 18 | ) 19 | def test_provider_aws(aws_provider_mock, azure_provider_mock, aws_token_mock): 20 | """Test provider is AWS.""" 21 | assert provider(["aws", "azure"]).identifier == "aws" 22 | 23 | 24 | @patch.object(AWSProvider, "identify", return_value=False) 25 | @patch.object(AzureProvider, "identify", return_value=True) 26 | @patch.object( 27 | AWSProvider, 28 | "_AWSProvider__request_token", 29 | return_value=(None), 30 | ) 31 | def test_provider_azure(aws_provider_mock, azure_provider_mock, aws_token_mock): 32 | """Test provider is Azure.""" 33 | assert provider(["aws", "azure"]).identifier == "azure" 34 | 35 | 36 | @patch.object(AWSProvider, "identify", return_value=False) 37 | @patch.object(AzureProvider, "identify", return_value=False) 38 | @patch.object( 39 | AWSProvider, 40 | "_AWSProvider__request_token", 41 | return_value=(None), 42 | ) 43 | def test_provider_not_aws_or_azure( 44 | aws_provider_mock, 45 | azure_provider_mock, 46 | aws_token_mock, 47 | ): 48 | """Test provider is unknown.""" 49 | assert provider(["aws", "azure"]).identifier == "unknown" 50 | 51 | 52 | @patch.object(AWSProvider, "identify", return_value=False) 53 | @patch.object(AzureProvider, "identify", return_value=False) 54 | @patch.object( 55 | AWSProvider, 56 | "_AWSProvider__request_token", 57 | return_value=(None), 58 | ) 59 | def test_none_provider(aws_provider_mock, azure_provider_mock, aws_token_mock): 60 | """Test provider is unknown.""" 61 | assert provider(None).identifier == "unknown" 62 | 63 | 64 | @pytest.mark.skipif(condition=True, reason="Test should be manually run.") 65 | def test_provider_detect(): 66 | """Test provider is unknown.""" 67 | assert provider(["aws", "azure"]).identifier == "unknown" 68 | -------------------------------------------------------------------------------- /tests/resources/config_with_status.yaml: -------------------------------------------------------------------------------- 1 | watchmaker_version: ">= 0.24.0.dev" 2 | 3 | all: 4 | - salt: 5 | admin_groups: null 6 | admin_users: null 7 | computer_name: null 8 | environment: null 9 | ou_path: null 10 | salt_content: null 11 | salt_states: Highstate 12 | user_formulas: 13 | # To add extra formulas, specify them as a map of 14 | # : 15 | # The is the name of the directory in the salt file_root 16 | # where the formula will be placed. The must be a zip 17 | # file, and the zip must contain a top-level directory that, itself, 18 | # contains the actual salt formula. To "overwrite" submodule formulas, 19 | # make sure matches submodule names. E.g.: 20 | # ash-linux-formula: https://s3.amazonaws.com/salt-formulas/ash-linux-formula-master.zip 21 | # scap-formula: https://s3.amazonaws.com/salt-formulas/scap-formula-master.zip 22 | 23 | linux: 24 | - yum: 25 | repo_map: 26 | # SaltEL6: 27 | - dist: 28 | - redhat 29 | - centos 30 | el_version: 6 31 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/2019.2.8/salt-reposync-el6.repo 32 | - dist: amazon 33 | el_version: 6 34 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/2019.2.8/salt-reposync-amzn.repo 35 | # SaltEL7: 36 | - dist: 37 | - redhat 38 | - centos 39 | el_version: 7 40 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3004.2/salt-reposync-el7-python3.repo 41 | # SaltEL8: 42 | - dist: 43 | - redhat 44 | - centos 45 | el_version: 8 46 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3004.2/salt-reposync-el8-python3.repo 47 | - salt: 48 | salt_debug_log: null 49 | install_method: yum 50 | bootstrap_source: null 51 | git_repo: null 52 | salt_version: null 53 | 54 | windows: 55 | - salt: 56 | salt_debug_log: null 57 | installer_url: https://watchmaker.cloudarmor.io/repo/saltstack/salt/windows/Salt-Minion-3004.2-1-Py3-AMD64-Setup.exe 58 | 59 | status: 60 | providers: 61 | - key: "WatchmakerStatus" 62 | required: false 63 | provider_type: "aws" 64 | - key: "WatchmakerStatus" 65 | required: false 66 | provider_type: "azure" 67 | -------------------------------------------------------------------------------- /ci/build.ps1: -------------------------------------------------------------------------------- 1 | $ErrorActionPreference = "Stop" 2 | 3 | $VERSION = (Select-String -Path pyproject.toml -Pattern '^version = ').Line -replace '^version = "(.+)".*$', '$1' 4 | 5 | $VIRTUAL_ENV_DIR = ".pyinstaller\venv" 6 | 7 | $PYI_DIST_DIR = ".pyinstaller\dist\${VERSION}" 8 | $PYI_SPEC_DIR = ".pyinstaller\spec" 9 | $PYI_WORK_DIR = ".pyinstaller\build" 10 | 11 | $PYI_HOOK_DIR = ".\ci\pyinstaller\win" 12 | $PYI_SCRIPT = "${PYI_SPEC_DIR}\watchmaker-standalone.py" 13 | $WAM_SCRIPT = ".\src\watchmaker\__main__.py" 14 | $WAM_FILENAME = "watchmaker-${VERSION}-standalone-windows-amd64" 15 | 16 | uv venv "$VIRTUAL_ENV_DIR" 17 | & "${VIRTUAL_ENV_DIR}\Scripts\Activate.ps1" 18 | 19 | Write-Host "-----------------------------------------------------------------------" 20 | python --version 21 | uv --version 22 | Write-Host "-----------------------------------------------------------------------" 23 | 24 | uv pip install -r requirements\build.txt 25 | uv pip install --editable . 26 | uv pip list 27 | 28 | Write-Host "Creating standalone for watchmaker v${VERSION}..." 29 | New-Item -Path "$PYI_SPEC_DIR" -Force -ItemType "directory" 30 | Copy-Item "$WAM_SCRIPT" -Destination "$PYI_SCRIPT" 31 | # Add debug argument to pyinstaller command to build standalone with debug flags 32 | # --debug all \ 33 | uv run pyinstaller --noconfirm --clean --onefile ` 34 | --name "$WAM_FILENAME" ` 35 | --runtime-tmpdir . ` 36 | --paths src ` 37 | --additional-hooks-dir "$PYI_HOOK_DIR" ` 38 | --distpath "$PYI_DIST_DIR" ` 39 | --specpath "$PYI_SPEC_DIR" ` 40 | --workpath "$PYI_WORK_DIR" ` 41 | "$PYI_SCRIPT" 42 | 43 | # Uncomment this to list the files in the standalone; can help when debugging 44 | # echo "Listing files in standalone..." 45 | # pyi-archive_viewer --log --brief --recursive "${DIST_DIR}/${WAM_FILENAME}.exe" 46 | 47 | Write-Host "Creating sha256 hashes of standalone binary..." 48 | $WAM_HASH = Get-FileHash -Algorithm SHA256 "${PYI_DIST_DIR}\${WAM_FILENAME}.exe" 49 | Set-Content -Path "${PYI_DIST_DIR}\${WAM_FILENAME}.exe.sha256" -Value "$($WAM_HASH.hash) ${WAM_FILENAME}.exe" 50 | Get-Content "${PYI_DIST_DIR}\${WAM_FILENAME}.exe.sha256" 51 | 52 | Write-Host "Checking standalone binary version..." 53 | & "${PYI_DIST_DIR}/${WAM_FILENAME}.exe" --version 54 | 55 | Write-Host "Copying bootstrap script to dist dirs..." 56 | Copy-Item docs/files/bootstrap/watchmaker-bootstrap.ps1 -Destination "$PYI_DIST_DIR" 57 | 58 | Write-Host "Listing files in dist dirs..." 59 | Get-ChildItem -Recurse -Force -Path "${PYI_DIST_DIR}\*" 60 | -------------------------------------------------------------------------------- /tests/resources/config_with_computer_name_pattern.yaml: -------------------------------------------------------------------------------- 1 | watchmaker_version: ">= 0.24.6.dev" 2 | 3 | all: 4 | - salt: 5 | admin_groups: null 6 | admin_users: null 7 | computer_name: null 8 | computer_name_pattern: (?i)xyz[\d]{3}[a-z]{8}[ex] 9 | environment: null 10 | ou_path: null 11 | salt_content: null 12 | salt_states: Highstate 13 | user_formulas: 14 | # To add extra formulas, specify them as a map of 15 | # : 16 | # The is the name of the directory in the salt file_root 17 | # where the formula will be placed. The must be a zip 18 | # file, and the zip must contain a top-level directory that, itself, 19 | # contains the actual salt formula. To "overwrite" submodule formulas, 20 | # make sure matches submodule names. E.g.: 21 | # ash-linux-formula: https://s3.amazonaws.com/salt-formulas/ash-linux-formula-master.zip 22 | # scap-formula: https://s3.amazonaws.com/salt-formulas/scap-formula-master.zip 23 | 24 | linux: 25 | - yum: 26 | repo_map: 27 | # SaltEL6: 28 | - dist: 29 | - redhat 30 | - centos 31 | el_version: 6 32 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/2019.2.8/salt-reposync-el6.repo 33 | - dist: amazon 34 | el_version: 6 35 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/2019.2.8/salt-reposync-amzn.repo 36 | # SaltEL7: 37 | - dist: 38 | - redhat 39 | - centos 40 | el_version: 7 41 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3004.2/salt-reposync-el7-python3.repo 42 | # SaltEL8: 43 | - dist: 44 | - redhat 45 | - centos 46 | el_version: 8 47 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3004.2/salt-reposync-el8-python3.repo 48 | - salt: 49 | salt_debug_log: null 50 | install_method: yum 51 | bootstrap_source: null 52 | git_repo: null 53 | salt_version: null 54 | 55 | windows: 56 | - salt: 57 | salt_debug_log: null 58 | installer_url: https://watchmaker.cloudarmor.io/repo/saltstack/salt/windows/Salt-Minion-3004.2-1-Py3-AMD64-Setup.exe 59 | 60 | status: 61 | providers: 62 | - key: "WatchmakerStatus" 63 | required: false 64 | provider_type: "aws" 65 | - key: "WatchmakerStatus" 66 | required: false 67 | provider_type: "azure" 68 | -------------------------------------------------------------------------------- /tests/resources/config_with_computer_name_and_pattern.yaml: -------------------------------------------------------------------------------- 1 | watchmaker_version: ">= 0.24.6.dev" 2 | 3 | all: 4 | - salt: 5 | admin_groups: null 6 | admin_users: null 7 | computer_name: abc321abcdefghe 8 | computer_name_pattern: (?i)abc[\d]{3}[a-z]{8}[ex] 9 | environment: null 10 | ou_path: null 11 | salt_content: null 12 | salt_states: Highstate 13 | user_formulas: 14 | # To add extra formulas, specify them as a map of 15 | # : 16 | # The is the name of the directory in the salt file_root 17 | # where the formula will be placed. The must be a zip 18 | # file, and the zip must contain a top-level directory that, itself, 19 | # contains the actual salt formula. To "overwrite" submodule formulas, 20 | # make sure matches submodule names. E.g.: 21 | # ash-linux-formula: https://s3.amazonaws.com/salt-formulas/ash-linux-formula-master.zip 22 | # scap-formula: https://s3.amazonaws.com/salt-formulas/scap-formula-master.zip 23 | 24 | linux: 25 | - yum: 26 | repo_map: 27 | # SaltEL6: 28 | - dist: 29 | - redhat 30 | - centos 31 | el_version: 6 32 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/2019.2.8/salt-reposync-el6.repo 33 | - dist: amazon 34 | el_version: 6 35 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/2019.2.8/salt-reposync-amzn.repo 36 | # SaltEL7: 37 | - dist: 38 | - redhat 39 | - centos 40 | el_version: 7 41 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3004.2/salt-reposync-el7-python3.repo 42 | # SaltEL8: 43 | - dist: 44 | - redhat 45 | - centos 46 | el_version: 8 47 | url: https://watchmaker.cloudarmor.io/yum.defs/saltstack/salt/3004.2/salt-reposync-el8-python3.repo 48 | - salt: 49 | salt_debug_log: null 50 | install_method: yum 51 | bootstrap_source: null 52 | git_repo: null 53 | salt_version: null 54 | 55 | windows: 56 | - salt: 57 | salt_debug_log: null 58 | installer_url: https://watchmaker.cloudarmor.io/repo/saltstack/salt/windows/Salt-Minion-3004.2-1-Py3-AMD64-Setup.exe 59 | 60 | status: 61 | providers: 62 | - key: "WatchmakerStatus" 63 | required: false 64 | provider_type: "aws" 65 | - key: "WatchmakerStatus" 66 | required: false 67 | provider_type: "azure" 68 | -------------------------------------------------------------------------------- /src/watchmaker/managers/worker_manager.py: -------------------------------------------------------------------------------- 1 | """Workers Manager module.""" 2 | 3 | import abc 4 | from typing import ClassVar 5 | 6 | from watchmaker.workers.salt import SaltLinux, SaltWindows 7 | from watchmaker.workers.yum import Yum 8 | 9 | 10 | class WorkersManagerBase(metaclass=abc.ABCMeta): 11 | """ 12 | Base class for worker managers. 13 | 14 | Args: 15 | system_params: (:obj:`dict`) 16 | Attributes, mostly file-paths, specific to the system-type (Linux 17 | or Windows). 18 | 19 | workers: (:obj:`collections.OrderedDict`) 20 | Workers to run and associated configuration data. 21 | 22 | """ 23 | 24 | WORKERS: ClassVar[dict] = {} 25 | 26 | def __init__(self, system_params, workers, *args, **kwargs): 27 | self.system_params = system_params 28 | self.workers = workers 29 | WorkersManagerBase.args = args 30 | WorkersManagerBase.kwargs = kwargs 31 | 32 | @abc.abstractmethod 33 | def _worker_execution(self): 34 | pass 35 | 36 | @abc.abstractmethod 37 | def _worker_validation(self): 38 | pass 39 | 40 | def worker_cadence(self): 41 | """Manage worker cadence.""" 42 | workers = [] 43 | 44 | for worker, items in self.workers.items(): 45 | configuration = items["config"] 46 | workers.append( 47 | self.WORKERS.get(worker)( 48 | system_params=self.system_params, 49 | **configuration, 50 | ), 51 | ) 52 | 53 | for worker in workers: 54 | worker.before_install() 55 | 56 | for worker in workers: 57 | worker.install() 58 | 59 | @abc.abstractmethod 60 | def cleanup(self): # noqa: D102 61 | pass 62 | 63 | 64 | class LinuxWorkersManager(WorkersManagerBase): 65 | """Manage the worker cadence for Linux systems.""" 66 | 67 | WORKERS: ClassVar[dict] = {"yum": Yum, "salt": SaltLinux} 68 | 69 | def _worker_execution(self): 70 | pass 71 | 72 | def _worker_validation(self): 73 | pass 74 | 75 | def cleanup(self): 76 | """Execute cleanup function.""" 77 | 78 | 79 | class WindowsWorkersManager(WorkersManagerBase): 80 | """Manage the worker cadence for Windows systems.""" 81 | 82 | WORKERS: ClassVar[dict] = {"salt": SaltWindows} 83 | 84 | def _worker_execution(self): 85 | pass 86 | 87 | def _worker_validation(self): 88 | pass 89 | 90 | def cleanup(self): 91 | """Execute cleanup function.""" 92 | -------------------------------------------------------------------------------- /docs/troubleshooting/Windows/c_watchmaker_logs_salt_call.debug.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # The `c:\watchmaker\logs\salt_call.debug` Log-File 11 | 12 | This file captures the execution-activities of [SaltStack](https://docs.saltproject.io/en/latest/contents.html) formulae. This file will exist if `watchmaker` has be able to successfully download and install its (SaltStack-based) configuration-content. 13 | 14 | The primary diagnostic interest in this file is if there is an execution-failure within a managed-content module. By default, `watchmaker` will reboot a system after a successful run[^1]. If the expected reboot occurs, this file likely will not be of interest. If the reboot fails to occur and the `watchmaker` log indicates that it was able to start the SaltStack-based operations, then consult this file to identify what failed and (possibly) why. 15 | 16 | ## Typical Errors 17 | 18 | Any errors encountered by SaltStack will typically have a corresponding log-section that starts with a string like: 19 | 20 | ``` 21 | 2023-06-27 12:57:39,841 [salt.state :325 ][ERROR ][5656] { ... } 22 | ``` 23 | 24 | Errors from the failing SaltStack action will typically include an embedded JSON-stream. The above snippet's `{ ... }` stands in for an embedded JSON-stream (for brevity's sake). Depending how long the embedded JSON-stream is, it will probably make things easier for the provisioning-user to convert that stream to a more human-readable JSON document-block. 25 | 26 | The most commonly-reported issues are around: 27 | 28 | - Domain Join Errors 29 | 30 | ### Domain Join Error 31 | 32 | Errors in joining the host to active directory can have several causes. The three most typical are: 33 | 34 | - Bad join-user credentials (or locked-out account) 35 | - Inability to find domain controllers 36 | - Inability to communicate with found domain controllers. 37 | 38 | The following is version of the salt_call.debug log file with a join-domain failure. The version shown has the JSON-stream expanded into a (more-readable) JSON-document. The original content can [be viewed](salt.debug.join-fail-stream.txt) to illustrate _why_ expanding the JSON-stream makes the provisioning-administrator's life easier. 39 | 40 | 41 | ```{eval-rst} 42 | .. literalinclude:: salt.debug.join-fail-expanded.txt 43 | :language: text 44 | :emphasize-lines: 228-232,253-257,278-282 45 | ``` 46 | 47 | 48 | 49 | 50 | [^1]: This behavior may be overridden by having invoked `watchmaker` with the `-n` flag 51 | -------------------------------------------------------------------------------- /docs/troubleshooting/Windows/index.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # Windows Log-Files 11 | 12 | When using `watchmaker` on Windows servers, the primary log-files of interest are: 13 | 14 | ```{toctree} 15 | :maxdepth: 1 16 | c_watchmaker_logs_watchmaker.log.md 17 | c_watchmaker_logs_salt_call.debug.md 18 | ``` 19 | 20 | There can be other files in the `c:\watchmaker\logs\` directory, but the ones present will depend on what enterprise-integration features have been selected for `watchmaker` to attempt to execute and whether those integrations are configured to log independently. 21 | 22 | There may be further log-files of interest, depending on how much execution-progress `watchmaker` has made and how `watchmaker` has been invoked. These will typically vary by build environment (e.g., when used with a CSP like Azure or AWS, on a physical server or a VM) and what tooling was used to invoke watchmaker. 23 | 24 | The additional log-files of interest are typically generated by whatever Windows-specific userData payload-handler is leveraged. The known additonal log-files of interest will be enumerated in further sub-sections. If you are using watchmaker via userData payload and the handler is not enumerated below, please contribute to this project's documentation. 25 | 26 | ## AWS: 27 | 28 | When official Windows instances – ones published through the Amazon/Microsoft partnership – are launched into AWS and execute watchmaker via a userData payload, either of the following log files will be created: 29 | 30 | - `C:\ProgramData\Amazon\EC2Launch\log\agent.log` (_see: ["EC2Launch" discussion-document](c_amazon_EC2Launch_Log_UserdataExecution.log.md)_) 31 | - `C:\ProgramData\Amazon\EC2-Windows\Launch\Log\UserdataExecution.log` (_see: ["EC2Launch v2" discussion- document](c_amazon_EC2Launch_v2_Log_UserdataExecution.log.md)_) 32 | 33 | _Which_ log file gets created will depend on the userData-handler used. Older versions of Windows Server (2012, 2016 and 2019) typically use the EC2 Launch handler. Newer versions of Windows Server (2022) use the EC2 v2 Launch-handler[^1]. 34 | 35 | 36 | ```{toctree} 37 | :maxdepth: 1 38 | :hidden: 39 | c_amazon_EC2Launch_Log_UserdataExecution.log.md 40 | c_amazon_EC2Launch_v2_Log_UserdataExecution.log.md 41 | ``` 42 | 43 | [^1]: Since the introduction of the EC2 v2 Launch-handler, official Windows Server AMIs for Server 2012, 2016 and 2019 have been being published. However, they are not the current default (as of the writing of this document). See the link to the EC2 v2 Launch discussion-document for details and caveats. 44 | -------------------------------------------------------------------------------- /.github/workflows/integration.yml: -------------------------------------------------------------------------------- 1 | name: Run terrafirm integration tests 2 | 3 | on: 4 | # Run on demand 5 | workflow_dispatch: 6 | 7 | # Run on pull request review with a specific command 8 | pull_request_review: 9 | types: [submitted] 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | trigger: 16 | runs-on: ubuntu-latest 17 | if: contains(github.event.review.body, '/build') || github.event_name == 'workflow_dispatch' 18 | outputs: 19 | run-id: ${{ steps.trigger.outputs.run-id }} 20 | steps: 21 | - name: Set terrafirm run-id 22 | id: trigger 23 | run: | 24 | RUN_ID=$(uuidgen) 25 | echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT" 26 | echo "RUN_ID=${RUN_ID}" 27 | 28 | test-source: 29 | runs-on: 30 | - codebuild-p3-terrafirm-${{ github.run_id }}-${{ github.run_attempt }} 31 | instance-size:small 32 | needs: trigger 33 | strategy: 34 | fail-fast: false 35 | matrix: 36 | source-build: [rhel8, rhel9, win16, win19, win22] 37 | env: 38 | AWS_DEFAULT_REGION: us-east-1 39 | TF_VAR_aws_region: us-east-1 40 | TF_VAR_codebuild_id: ${{ needs.trigger.outputs.run-id }} 41 | TF_VAR_common_args: "-n -e dev" 42 | TF_VAR_git_ref: ${{ github.ref || github.sha }} 43 | TF_VAR_git_repo: "${{ github.server_url }}/${{ github.repository }}.git" 44 | TF_VAR_source_builds: '["${{ matrix.source-build }}"]' 45 | TF_VAR_standalone_builds: '[]' 46 | steps: 47 | - name: Terrafirm integration tests 48 | id: terrafirm 49 | uses: plus3it/terrafirm/.github/actions/test@d4283972b2e3738a6ad61a43225a0a5b71d4e83a 50 | with: 51 | destroy-after-test: true 52 | 53 | test-standalone: 54 | runs-on: 55 | - codebuild-p3-terrafirm-${{ github.run_id }}-${{ github.run_attempt }} 56 | instance-size:small 57 | needs: trigger 58 | strategy: 59 | fail-fast: false 60 | matrix: 61 | standalone-build: [rhel8, rhel9, win16, win19, win22] 62 | env: 63 | AWS_DEFAULT_REGION: us-east-1 64 | TF_VAR_aws_region: us-east-1 65 | TF_VAR_codebuild_id: ${{ needs.trigger.outputs.run-id }} 66 | TF_VAR_common_args: "-n -e dev" 67 | TF_VAR_git_ref: ${{ github.ref || github.sha }} 68 | TF_VAR_git_repo: "${{ github.server_url }}/${{ github.repository }}.git" 69 | TF_VAR_source_builds: '[]' 70 | TF_VAR_standalone_builds: '["${{ matrix.standalone-build }}"]' 71 | steps: 72 | - name: Terrafirm integration tests 73 | id: terrafirm 74 | uses: plus3it/terrafirm/.github/actions/test@d4283972b2e3738a6ad61a43225a0a5b71d4e83a 75 | with: 76 | destroy-after-test: true 77 | -------------------------------------------------------------------------------- /docs/gotchas/EL8-OpenSSHkeyLogins.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # OpenSSH RSAv2 Keys Don't Work (EL8) 11 | 12 | ## Background 13 | 14 | The OpenSSH Daemon shipped with the most-recent versions of RHEL 8 (and derivatives), implements the deprecation of SHA1-signed SSH keys for key-based authentication that's now part of OpenSSH 8.8 and higher. As such, any SSH keys used for key-based authentication will need to be signed using a SHA2 algorithm (SHA-256 or SHA-512). 15 | 16 | ## Workarounds 17 | 18 | For users of self-managed keys, this means that one needs to present an SHA-256 or SHA-512 signed OpenSSH key when using RSAv2 keys for key-based logins. Such keys can be generated in a couple ways: 19 | 20 | * Use either `rsa-sha2-256` or `rsa-sha2-512` when using `ssh-keygen`'s `-t` option for generating a new key 21 | * Use `ssh-keygen` on a FIPS-enabled, EL8+ operating system 22 | * Use a CSP's key-generation tool (AWS's commercial region's EC2 key-generation capability is known to create conformant RSAv2 keys) 23 | 24 | For users of organizationally-issued SSH keys - be they bare files or as delivered via a centrally-managed SmartCard (such as a PIV or CAC) or other token - it will be necessary for the key-user to work with their organization to ensure that updated, conformant keys are issued. 25 | 26 | ## Symptoms 27 | 28 | Depending on the SSH client, the key may silently fail to work or it may print an error. If an error is printed, it will usually be something like: 29 | 30 | ```bash 31 | Load key "/path/to/key-file": error in libcrypto 32 | ``` 33 | 34 | With or without the printing of the error, the key will be disqualified and the server will request the client move on to the next-available authentication-metho (usually password). 35 | 36 | _If_ one is able to use other means to access a system and view its logs, one will usually find errors similar to: 37 | 38 | ```bash 39 | Feb 09 12:10:50 ip-0a00dc73 sshd[2939]: input_userauth_request: invalid user ec2-user [preauth] 40 | ``` 41 | 42 | Or 43 | 44 | ```bash 45 | Feb 09 12:10:50 ip-0a00dc73 sshd[2939]: input_userauth_pubkey: key type ssh-rsa not in PubkeyAcceptedKeyTypes [preauth] 46 | ``` 47 | 48 | In the `/var/log/secure` logs. 49 | 50 | 51 | 52 | **Note:** The deprecated SHA-1 issuse is not a watchmaker issue. It is generically applicable to Red Hat's OpenSSH version on EL8-bsed systems. However, because most people will encounter the issue after having run watchmaker, we opted to include it in this project's "Gotchas" documentation for the benefit of watchmaker-users that might come here for answers 53 | -------------------------------------------------------------------------------- /docs/troubleshooting/Linux/watchmaker.log.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # The `/var/log/watchmaker/watchmaker.log` Log-File 11 | 12 | This file tracks the top-level execution of the `watchmaker` configuration-utility. This file should always exist, unless: 13 | 14 | - The provisioning-administrator has checked for the log before the utility has been downloaded and an execution-attempted. This typically happens if a `watchmaker`-execution is attempted late in a complex provisioning-process 15 | - An execution-attempt wholly failed. In this case, check the logs for the `watchmaker`-calling service or process (e.g. [`cloud-init`](cloud-init.log.md)) 16 | - The provisioning-administrator has not invoked `watchmaker` in accordance with the `watchmaker` project's usage-guidance: if a different logging-location was specified (e.g., by adding a flag/argument like `--log-dir=/tmp/watchmaker`), the provisioning-administrator would need to check the alternately-specified logging-location. 17 | - The provisioning-administrator invoked the `watchmaker`-managed _content_ directly (e.g., using `salt-call -c /srv/watchmaker/salt`). In this scenario, only the content-execution may have been logged (whether logging was captured and where would depend on how the direct-execution was requested). 18 | 19 | ## Typical Errors 20 | 21 | * Bad specification of remotely-hosted configuration file. This will typically come with an HTTP 404 error similar to: 22 | ~~~ 23 | botocore.exceptions.ClientError: An error occurred (404) when calling the HeadObject operation: Not Found 24 | ~~~ 25 | 26 | Ensure that the requested URI for the remotely-hosted configuration file is valid. 27 | * Attempt to use a protected, remotely-hosted configuration-file. This will typically come win an HTTP 403 error. Most typically, this happens when the requested configuration-file exists on a protected network share and the requesting-process doesn't have permission to access it. 28 | ~~~ 29 | botocore.exceptions.ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden 30 | ~~~ 31 | 32 | Ensure that `watchmaker` has adequate permissions to access the requested, remotely-hosted configuration file. 33 | * Remotely-hosted configuration file is specified as an `s3://` URI without installation of `boto3` Python module. This will typically come with an error similar to: 34 | ```{eval-rst} 35 | .. literalinclude:: ../NoBoto3-LogSnippet.txt 36 | :language: text 37 | :emphasize-lines: 1-2 38 | ``` 39 | 40 | Ensure that the `boto3` Python module has been installed _prior to_ attempting to execute `watchmaker` 41 | -------------------------------------------------------------------------------- /tests/imds-providers/test_aws_provider.py: -------------------------------------------------------------------------------- 1 | """Providers main test module.""" 2 | 3 | import json 4 | from unittest.mock import patch 5 | 6 | from watchmaker.utils.imds.detect.providers.aws_provider import AWSProvider 7 | 8 | 9 | @patch.object( 10 | AWSProvider, 11 | "_AWSProvider__call_urlopen_retry", 12 | return_value=(json.dumps({"imageId": "ami-12312412", "instanceId": "i-ec12as"})), 13 | ) 14 | @patch.object( 15 | AWSProvider, 16 | "_AWSProvider__request_token", 17 | return_value=(None), 18 | ) 19 | def test_identify_is_valid(mock_urlopen, mock_request_token): 20 | """Test valid server data response for aws provider identification.""" 21 | provider = AWSProvider() 22 | assert provider.identify() is True 23 | 24 | 25 | @patch.object( 26 | AWSProvider, 27 | "_AWSProvider__call_urlopen_retry", 28 | return_value=(json.dumps({"imageId": "ami-12312412", "instanceId": "i-ec12as"})), 29 | ) 30 | @patch.object( 31 | AWSProvider, 32 | "_AWSProvider__request_token", 33 | return_value=(None), 34 | ) 35 | def test_metadata_server_is_valid(mock_urlopen, mock_request_token): 36 | """Test valid server data response for aws provider identification.""" 37 | provider = AWSProvider() 38 | assert provider.check_metadata_server() is True 39 | 40 | 41 | @patch.object( 42 | AWSProvider, 43 | "_AWSProvider__call_urlopen_retry", 44 | return_value=(json.dumps({"imageId": "not-valid", "instanceId": "etc-ec12as"})), 45 | ) 46 | @patch.object( 47 | AWSProvider, 48 | "_AWSProvider__request_token", 49 | return_value=(None), 50 | ) 51 | def test_metadata_server_is_invalid(mock_urlopen, mock_request_token): 52 | """Test invalid server data response for aws provider identification.""" 53 | provider = AWSProvider() 54 | assert provider.check_metadata_server() is False 55 | 56 | 57 | @patch.object( 58 | AWSProvider, 59 | "_AWSProvider__request_token", 60 | return_value=("abcdefgh1234546"), 61 | ) 62 | @patch.object( 63 | AWSProvider, 64 | "_AWSProvider__call_urlopen_retry", 65 | return_value=(None), 66 | ) 67 | def test_aws_metadata_headers(mock_request_token, mock_urlopen): 68 | """Test token is not saved to imds_token.""" 69 | provider = AWSProvider() 70 | assert provider.get_metadata_request_headers() == { 71 | "X-aws-ec2-metadata-token": "abcdefgh1234546", 72 | } 73 | 74 | 75 | @patch.object( 76 | AWSProvider, 77 | "_AWSProvider__request_token", 78 | return_value=(None), 79 | ) 80 | @patch.object( 81 | AWSProvider, 82 | "_AWSProvider__call_urlopen_retry", 83 | return_value=(None), 84 | ) 85 | def test_aws_metadata_headers_none(mock_request_token, mock_urlopen): 86 | """Test token is not saved to imds_token.""" 87 | provider = AWSProvider() 88 | assert provider.get_metadata_request_headers() is None 89 | -------------------------------------------------------------------------------- /tests/status/test_status.py: -------------------------------------------------------------------------------- 1 | """Providers main test module.""" 2 | 3 | from unittest.mock import patch 4 | 5 | import pytest 6 | 7 | from watchmaker.exceptions import CloudProviderDetectionError 8 | from watchmaker.status import Status 9 | from watchmaker.utils.imds.detect.providers.aws_provider import AWSProvider 10 | from watchmaker.utils.imds.detect.providers.azure_provider import AzureProvider 11 | 12 | 13 | @patch.object(AWSProvider, "identify", return_value=True) 14 | @patch.object(AzureProvider, "identify", return_value=False) 15 | @patch( 16 | "watchmaker.config.status.get_cloud_with_prereqs", 17 | return_value=["aws", "azure"], 18 | ) 19 | @patch.object( 20 | AWSProvider, 21 | "_AWSProvider__request_token", 22 | return_value=(None), 23 | ) 24 | def test_status( 25 | aws_provider_mock, 26 | azure_provider_mock, 27 | supported_identifiers_mock, 28 | request_token_mock, 29 | ): 30 | """Test provider is AWS.""" 31 | config = { 32 | "status": { 33 | "providers": [ 34 | {"key": "WatchmakerStatus", "required": False, "provider_type": "aws"}, 35 | { 36 | "key": "WatchmakerStatus", 37 | "required": False, 38 | "provider_type": "azure", 39 | }, 40 | ], 41 | }, 42 | } 43 | config_status = config.get("status") 44 | status = Status(config_status) 45 | detected_providers = status.get_detected_status_providers() 46 | assert len(detected_providers) == 1 47 | assert detected_providers.get("aws").identifier == AWSProvider.identifier 48 | 49 | 50 | @patch.object(AWSProvider, "identify", return_value=False) 51 | @patch.object(AzureProvider, "identify", return_value=True) 52 | @patch( 53 | "watchmaker.config.status.get_cloud_with_prereqs", 54 | return_value=[], 55 | ) 56 | @patch( 57 | "watchmaker.config.status.get_cloud_missing_prereqs", 58 | return_value=["aws", "azure"], 59 | ) 60 | @patch.object( 61 | AWSProvider, 62 | "_AWSProvider__request_token", 63 | return_value=(None), 64 | ) 65 | def test_req_status_provider( 66 | aws_provider_mock, 67 | azure_provider_mock, 68 | supported_identifiers_mock, 69 | missing_prereqs_mock, 70 | request_token_mock, 71 | ): 72 | """Test provider is AWS.""" 73 | config = { 74 | "status": { 75 | "providers": [ 76 | {"key": "WatchmakerStatus", "required": False, "provider_type": "aws"}, 77 | {"key": "WatchmakerStatus", "required": True, "provider_type": "azure"}, 78 | ], 79 | }, 80 | } 81 | config_status = config.get("status") 82 | 83 | with pytest.raises( 84 | CloudProviderDetectionError, 85 | match="Required Provider detected that is missing prereqs: azure", 86 | ): 87 | Status(config_status) 88 | -------------------------------------------------------------------------------- /docs/customization/index.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # Execution Customization 11 | 12 | This document is intended to help both the `watchmaker` user-community and `watchmaker` developers and contributors better understand how to customize the execution of the `watchmaker` configuration-utility. This will be covered in the documents linked-to from the (below) "**Common Scenarios**" section. 13 | 14 | ## Background 15 | 16 | By default, `watchmaker` executes a standard set of configuration-tasks. The `watchmaker` utility primarily leverages [SaltStack](https://docs.saltproject.io/en/latest/topics/about_salt_project.html#about-salt) for these configuration-tasks. 17 | 18 | The configuration-tasks, themselves, are grouped into sets of related tasks. Related tasks can be things like: 19 | 20 | - Performing OS-hardening (e.g., applying STIGs) 21 | - Joining a Linux or Windows host to an Active Directory domain 22 | - Installing/configuring enterprise-mandated software (e.g., anti-virus or other security-tooling) 23 | - etc. 24 | 25 | These task-sets are delivered in the form of formulas. From the [vendor documentation](https://docs.saltproject.io/en/latest/topics/development/conventions/formulas.html) on formulas: 26 | 27 | > Formulas are pre-written Salt States. They are as open-ended as Salt States themselves and can be used for tasks such as installing a package, configuring, and starting a service, setting up users or permissions, and many other common tasks. 28 | > 29 | > All official Salt Formulas are found as separate Git repositories in the "saltstack-formulas" organization on GitHub 30 | 31 | The `watchmaker` project follows a similar convention. Formulae specifically authored to work under `watchmaker` can be found by visiting [Plus3 IT's GitHub](https://github.com/plus3it) and querying for the substring, "[-formula](https://github.com/plus3it/?q=-formula&type=all&language=&sort=)". 32 | 33 | ## Critical Files 34 | 35 | Customization-activities will be governed by two, main files: the watchmaker configuration file (a.k.a.,`config.yaml`) and the Salt content archive (a.k.a., `salt-content.zip`). Discussion of the files' contents are as follows: 36 | 37 | ```{toctree} 38 | :maxdepth: 1 39 | ConfigYaml.md 40 | SaltContent.md 41 | ``` 42 | 43 | ## Common Scenarios 44 | 45 | The behavior of watchmaker can be easily customized towards several ends. The most-commonly encountered are: 46 | 47 | ```{toctree} 48 | :maxdepth: 1 49 | 50 | SiteParameters.md 51 | SiteFormulae.md 52 | FormulaUpdates.md 53 | NewFormulas.md 54 | ``` 55 | 56 | If there are other customization-scenarios that should be included in this document-set, please see the [contribution guidance](../contributing). The contribution document covers how to submit requests for documentation-improvements as well as guidance on how to contribute changes (like further customization documentation). 57 | -------------------------------------------------------------------------------- /ci/prep_docker_pyapp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu -o pipefail 4 | 5 | echo "***********************************************************************" 6 | echo "Prepping for Docker PyApp Watchmaker build on $(lsb_release -ds 2>/dev/null || echo "Unknown Linux")" 7 | echo "***********************************************************************" 8 | 9 | DOCKER_INSTANCE_NAME=wam-pyapp-builder 10 | echo "DOCKER_INSTANCE_NAME:${DOCKER_INSTANCE_NAME}" 11 | 12 | WORKDIR="${WORKDIR:-${TRAVIS_BUILD_DIR:-}}" 13 | WORKDIR="${WORKDIR:-${PWD}}" 14 | echo "WORKDIR:${WORKDIR}" 15 | 16 | # Extract uv version from requirements/basics.txt 17 | UV_VERSION=$(grep -E '^uv==' "${WORKDIR}/requirements/basics.txt" | sed 's/^uv==//') 18 | echo "UV_VERSION:${UV_VERSION}" 19 | 20 | if [ -n "${WORKDIR}" ]; then 21 | 22 | # Clean up any existing container with the same name 23 | if docker ps -a --format '{{.Names}}' | grep -q "^${DOCKER_INSTANCE_NAME}$"; then 24 | echo "Removing existing container ${DOCKER_INSTANCE_NAME}..." 25 | docker stop "${DOCKER_INSTANCE_NAME}" || true 26 | docker rm "${DOCKER_INSTANCE_NAME}" 27 | fi 28 | 29 | echo "Building Docker container..." 30 | docker build \ 31 | --build-arg USER_UID="$(id -u)" \ 32 | --build-arg USER_GID="$(id -g)" \ 33 | --build-arg UV_VERSION="${UV_VERSION}" \ 34 | -t "$DOCKER_INSTANCE_NAME" -f "${WORKDIR}/ci/Dockerfile.pyapp" . 35 | 36 | # Detect if using podman and set appropriate userns flag 37 | USERNS_FLAG="" 38 | if command -v podman &> /dev/null && docker --version 2>&1 | grep -qi podman; then 39 | echo "Detected Podman, using --userns=keep-id" 40 | USERNS_FLAG="--userns=keep-id" 41 | fi 42 | 43 | # Setup cargo cache directory local to project (for dependencies and build artifacts) 44 | CARGO_CACHE_DIR="${WORKDIR}/.pyapp/build/.cargo-cache" 45 | mkdir -p "${CARGO_CACHE_DIR}" 46 | echo "Using Cargo cache directory: ${CARGO_CACHE_DIR}" 47 | 48 | # Setup local/share directory for uv cache and pyapp runtime data 49 | LOCAL_SHARE_DIR="${WORKDIR}/.pyapp/build/.local-share" 50 | mkdir -p "${LOCAL_SHARE_DIR}" 51 | echo "Using local/share directory for caches: ${LOCAL_SHARE_DIR}" 52 | 53 | echo "Building using container and workdir (${WORKDIR})..." 54 | docker run --detach \ 55 | --user "$(id -u):$(id -g)" \ 56 | ${USERNS_FLAG} \ 57 | --volume="${WORKDIR}:${WORKDIR}" \ 58 | --volume="${CARGO_CACHE_DIR}:/home/wam-builder/.cargo/registry" \ 59 | --volume="${LOCAL_SHARE_DIR}:/home/wam-builder/.local/share" \ 60 | --workdir "${WORKDIR}" \ 61 | --name "${DOCKER_INSTANCE_NAME}" \ 62 | "${DOCKER_INSTANCE_NAME}:latest" \ 63 | init 64 | 65 | # Ensure container cleanup happens on exit 66 | cleanup() { 67 | echo "Stopping the docker container ${DOCKER_INSTANCE_NAME}..." 68 | docker stop "${DOCKER_INSTANCE_NAME}" 2>/dev/null || true 69 | echo "Removing the docker container ${DOCKER_INSTANCE_NAME}..." 70 | docker rm "${DOCKER_INSTANCE_NAME}" 2>/dev/null || true 71 | } 72 | trap cleanup EXIT 73 | 74 | echo "Building the standalone using ci/build_pyapp.sh..." 75 | docker exec "${DOCKER_INSTANCE_NAME}" chmod +x ci/build_pyapp.sh 76 | docker exec "${DOCKER_INSTANCE_NAME}" ci/build_pyapp.sh 77 | 78 | else 79 | 80 | echo "No WORKDIR provided so not building..." 81 | 82 | fi 83 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "src/watchmaker/static/salt/formulas/ash-linux-formula"] 2 | path = src/watchmaker/static/salt/formulas/ash-linux-formula 3 | url = https://github.com/plus3it/ash-linux-formula.git 4 | [submodule "src/watchmaker/static/salt/formulas/ash-windows-formula"] 5 | path = src/watchmaker/static/salt/formulas/ash-windows-formula 6 | url = https://github.com/plus3it/ash-windows-formula.git 7 | [submodule "src/watchmaker/static/salt/formulas/join-domain-formula"] 8 | path = src/watchmaker/static/salt/formulas/join-domain-formula 9 | url = https://github.com/plus3it/join-domain-formula.git 10 | [submodule "src/watchmaker/static/salt/formulas/splunkforwarder-formula"] 11 | path = src/watchmaker/static/salt/formulas/splunkforwarder-formula 12 | url = https://github.com/plus3it/splunkforwarder-formula.git 13 | [submodule "src/watchmaker/static/salt/formulas/mcafee-agent-formula"] 14 | path = src/watchmaker/static/salt/formulas/mcafee-agent-formula 15 | url = https://github.com/plus3it/mcafee-agent-formula.git 16 | [submodule "src/watchmaker/static/salt/formulas/name-computer-formula"] 17 | path = src/watchmaker/static/salt/formulas/name-computer-formula 18 | url = https://github.com/plus3it/name-computer-formula.git 19 | [submodule "src/watchmaker/static/salt/formulas/windows-update-agent-formula"] 20 | path = src/watchmaker/static/salt/formulas/windows-update-agent-formula 21 | url = https://github.com/plus3it/windows-update-agent-formula.git 22 | [submodule "src/watchmaker/static/salt/formulas/netbanner-formula"] 23 | path = src/watchmaker/static/salt/formulas/netbanner-formula 24 | url = https://github.com/plus3it/netbanner-formula.git 25 | [submodule "src/watchmaker/static/salt/formulas/ntp-client-windows-formula"] 26 | path = src/watchmaker/static/salt/formulas/ntp-client-windows-formula 27 | url = https://github.com/plus3it/ntp-client-windows-formula.git 28 | [submodule "src/watchmaker/static/salt/formulas/scap-formula"] 29 | path = src/watchmaker/static/salt/formulas/scap-formula 30 | url = https://github.com/plus3it/scap-formula.git 31 | [submodule "src/watchmaker/static/salt/formulas/pshelp-formula"] 32 | path = src/watchmaker/static/salt/formulas/pshelp-formula 33 | url = https://github.com/plus3it/pshelp-formula.git 34 | [submodule "src/watchmaker/static/salt/formulas/nessus-agent-formula"] 35 | path = src/watchmaker/static/salt/formulas/nessus-agent-formula 36 | url = https://github.com/plus3it/nessus-agent-formula.git 37 | [submodule "src/watchmaker/static/salt/formulas/amazon-inspector-formula"] 38 | path = src/watchmaker/static/salt/formulas/amazon-inspector-formula 39 | url = https://github.com/plus3it/amazon-inspector-formula.git 40 | [submodule "src/watchmaker/static/salt/content"] 41 | path = src/watchmaker/static/salt/content 42 | url = https://github.com/plus3it/watchmaker-salt-content.git 43 | [submodule "src/watchmaker/static/salt/formulas/fup-formula"] 44 | path = src/watchmaker/static/salt/formulas/fup-formula 45 | url = https://github.com/plus3it/fup-formula.git 46 | [submodule "src/watchmaker/static/salt/formulas/vault-auth-formula"] 47 | path = src/watchmaker/static/salt/formulas/vault-auth-formula 48 | url = https://github.com/plus3it/vault-auth-formula 49 | [submodule "src/watchmaker/static/salt/formulas/forescout-secure-connector-formula"] 50 | path = src/watchmaker/static/salt/formulas/forescout-secure-connector-formula 51 | url = https://github.com/plus3it/forescout-secure-connector-formula.git 52 | -------------------------------------------------------------------------------- /src/watchmaker/utils/imds/detect/providers/aws_provider.py: -------------------------------------------------------------------------------- 1 | """AWS Provider.""" 2 | 3 | import json 4 | import logging 5 | import os 6 | from urllib import error as urllib_error 7 | 8 | from watchmaker import utils 9 | from watchmaker.utils.imds.detect.providers.provider import AbstractProvider 10 | 11 | IMDS_TOKEN_TIMEOUT = int(os.getenv("IMDS_TOKEN_TIMEOUT", "7200")) 12 | 13 | 14 | class AWSProvider(AbstractProvider): 15 | """Concrete implementation of the AWS cloud provider.""" 16 | 17 | identifier = "aws" 18 | 19 | def __init__(self, logger=None): 20 | self.logger = logger or logging.getLogger(__name__) 21 | self.metadata_url = ( 22 | "http://169.254.169.254/latest/dynamic/instance-identity/document" 23 | ) 24 | 25 | self.metadata_imds_v2_token_url = "http://169.254.169.254/latest/api/token" # noqa: S105 26 | 27 | self.imds_token = self.__request_token() 28 | 29 | def identify(self): 30 | """Identify AWS using all the implemented options.""" 31 | self.logger.info("Try to identify AWS") 32 | return self.check_metadata_server() 33 | 34 | def check_metadata_server(self): 35 | """Identify AWS via metadata server.""" 36 | self.logger.debug("Checking AWS metadata") 37 | try: 38 | return self.__is_valid_server() 39 | except Exception: 40 | self.logger.exception("Unexpected error checking AWS metadata server") 41 | return False 42 | 43 | def get_metadata_request_headers(self): 44 | """Return metadata request header if imds token is set.""" 45 | if self.imds_token: 46 | self.logger.debug("Returning AWS IMDSv2 Token Header") 47 | return {"X-aws-ec2-metadata-token": self.imds_token} 48 | 49 | self.logger.debug("AWS IMDSv2 Token not found") 50 | return None 51 | 52 | def __is_valid_server(self): 53 | """Determine if valid metadata server.""" 54 | data = self.__call_urlopen_retry( 55 | self.metadata_url, 56 | self.DEFAULT_TIMEOUT, 57 | headers=self.get_metadata_request_headers(), 58 | ) 59 | 60 | if data: 61 | response = json.loads(data) 62 | if response["imageId"].startswith( 63 | "ami-", 64 | ) and response["instanceId"].startswith("i-"): 65 | return True 66 | return False 67 | 68 | def __request_token(self): 69 | try: 70 | self.logger.debug("Create request for token") 71 | return self.__call_urlopen_retry( 72 | self.metadata_imds_v2_token_url, 73 | self.DEFAULT_TIMEOUT, 74 | headers={"X-aws-ec2-metadata-token-ttl-seconds": IMDS_TOKEN_TIMEOUT}, 75 | method="PUT", 76 | ) 77 | except urllib_error.URLError as error: 78 | self.logger.debug("Failed to set IMDSv2 token: %s", error) 79 | return None 80 | 81 | def __call_urlopen_retry(self, uri, timeout, headers=None, method=None): 82 | request_uri = utils.urllib_utils.request.Request( 83 | uri, 84 | data=None, 85 | headers=headers, 86 | method=method, 87 | ) 88 | 89 | result = utils.urlopen_retry(request_uri, timeout) 90 | http_ok = 200 91 | if result.status == http_ok: 92 | return result.read().decode("utf-8") 93 | return None 94 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | """Providers main test module.""" 2 | 3 | import re 4 | from pathlib import Path 5 | from unittest.mock import patch 6 | 7 | import pytest 8 | 9 | from watchmaker.config import get_configs, validate_computer_name_pattern 10 | from watchmaker.exceptions import WatchmakerError 11 | 12 | 13 | def test_config_w_status_provider(): 14 | """Test config that has the status block and supported provider.""" 15 | with patch("watchmaker.utils.imds.detect.provider", return_value="aws"): 16 | config, status_config = get_configs( 17 | "linux", 18 | {}, 19 | Path("tests") / "resources" / "config_with_status.yaml", 20 | ) 21 | assert config is not None 22 | assert status_config is not None 23 | 24 | 25 | def test_config_wo_status_config(): 26 | """Test config that does not have status block or provider.""" 27 | with patch("watchmaker.utils.imds.detect.provider", return_value="unknown"): 28 | config, status_config = get_configs( 29 | "linux", 30 | {}, 31 | Path("tests") / "resources" / "config_without_status.yaml", 32 | ) 33 | assert config is not None 34 | assert status_config is None 35 | 36 | 37 | def test_config_w_name_pattern(): 38 | """Test config with name pattern compare valid/invalid names.""" 39 | valid_computer_name = "xyz654abcdefghe" 40 | invalid_computer_name = "123xyz654abcdefgheone" 41 | with patch("watchmaker.utils.imds.detect.provider", return_value="unknown"): 42 | config, _status_config = get_configs( 43 | "linux", 44 | {}, 45 | Path("tests") / "resources" / "config_with_computer_name_pattern.yaml", 46 | ) 47 | 48 | pattern = config["salt"]["config"]["computer_name_pattern"] 49 | 50 | assert pattern == r"(?i)xyz[\d]{3}[a-z]{8}[ex]" 51 | assert re.match(pattern + r"\Z", valid_computer_name) is not None 52 | assert re.match(pattern + r"\Z", invalid_computer_name) is None 53 | 54 | # Test with terminal patterns and supported \Z terminal pattern combined 55 | dbl_terminal_pattern = r"(?i)^xyz[\d]{3}[a-z]{8}[ex]" + r"$\Z" 56 | assert re.match(dbl_terminal_pattern, valid_computer_name) is not None 57 | assert re.match(dbl_terminal_pattern, invalid_computer_name) is None 58 | 59 | # Test without terminal patterns showing need for \Z 60 | assert re.match(pattern, valid_computer_name + "12345") is not None 61 | assert re.match(dbl_terminal_pattern, valid_computer_name + "12345") is None 62 | 63 | 64 | def test_config_w_name_and_pattern(): 65 | """Test config that has a pattern and computer name.""" 66 | with patch("watchmaker.utils.imds.detect.provider", return_value="unknown"): 67 | config, _status_config = get_configs( 68 | "linux", 69 | {}, 70 | str( 71 | Path("tests") 72 | / "resources" 73 | / "config_with_computer_name_and_pattern.yaml", 74 | ), 75 | ) 76 | computer_name = config["salt"]["config"]["computer_name"] 77 | pattern = config["salt"]["config"]["computer_name_pattern"] 78 | assert pattern == r"(?i)abc[\d]{3}[a-z]{8}[ex]" 79 | assert computer_name == "abc321abcdefghe" 80 | assert re.match(pattern + r"\Z", computer_name) is not None 81 | 82 | 83 | def test_config_validate_pattern(): 84 | """Test config validate pattern method.""" 85 | config, _status_config = get_configs( 86 | "linux", 87 | {}, 88 | str( 89 | Path("tests") / "resources" / "config_with_computer_name_and_pattern.yaml", 90 | ), 91 | ) 92 | 93 | validate_computer_name_pattern(config) 94 | config["salt"]["config"]["computer_name_pattern"] = r"?i)abc[\d]{3}[a-z]{8}[ex]" 95 | with pytest.raises(WatchmakerError): 96 | validate_computer_name_pattern(config) 97 | -------------------------------------------------------------------------------- /src/watchmaker/status/providers/azure.py: -------------------------------------------------------------------------------- 1 | """Azure Status Provider.""" 2 | 3 | import json 4 | import logging 5 | 6 | from watchmaker import utils 7 | from watchmaker.conditions import HAS_AZURE 8 | from watchmaker.exceptions import StatusProviderError 9 | from watchmaker.status.providers.abstract import AbstractStatusProvider 10 | 11 | 12 | class AzureStatusProvider(AbstractStatusProvider): 13 | """Concrete implementation of the Azure status cloud provider.""" 14 | 15 | identifier = "azure" 16 | 17 | def __init__(self, provider, logger=None): 18 | self.logger = logger or logging.getLogger(__name__) 19 | self.metadata_id_url = ( 20 | "http://169.254.169.254/metadata/instance?api-version=2021-02-01" 21 | ) 22 | self.initial_status = False 23 | self.subscription_id = None 24 | self.resource_id = None 25 | self.provider = provider 26 | self.initialize() 27 | 28 | def initialize(self): 29 | """Initialize subscription and resource id.""" 30 | if self.subscription_id and self.resource_id: 31 | return 32 | 33 | try: 34 | self.__set_ids_from_server() 35 | except Exception: 36 | self.logger.exception("Error retrieving ids from metadata service") 37 | 38 | def update_status(self, key, status, required): 39 | """Tag an Azure instance with the key and status provided.""" 40 | self.logger.debug("Tagging Azure Resource") 41 | if HAS_AZURE and self.subscription_id and self.resource_id and status: 42 | try: 43 | self.__tag_azure_resouce(key, status) 44 | except Exception: 45 | self.logger.exception("Exception while tagging azure resource") 46 | else: 47 | return 48 | self.__error_on_required_status(required) 49 | 50 | def __tag_azure_resouce(self, key, status): 51 | self.logger.debug("Tag Resource %s with %s:%s", self.resource_id, key, status) 52 | credential = AzureCliCredential() # noqa: F821 53 | 54 | resource_client = ResourceManagementClient( # noqa: F821 55 | credential, 56 | self.subscription_id, 57 | ) 58 | 59 | body = { 60 | "operation": self.__get_operation(), 61 | "properties": {"tags": {key: status}}, 62 | } 63 | resource_client.tags.create_or_update_at_scope(self.resource_id, body) 64 | self.logger.debug("Resource tag created") 65 | 66 | def __set_ids_from_server(self): 67 | """Retrieve Azure instance id from metadata.""" 68 | response = utils.urlopen_retry(self.metadata_id_url, self.DEFAULT_TIMEOUT) 69 | data = json.load(response) 70 | self.subscription_id = data["compute"]["subscriptionId"] 71 | self.resource_id = data["compute"]["resourceId"] 72 | 73 | def __get_operation(self): 74 | """ 75 | Get the tag operation. 76 | 77 | Return "create" if initial status otherwise "udpate" 78 | """ 79 | if self.initial_status: 80 | self.initial_status = False 81 | return "create" 82 | 83 | return "update" 84 | 85 | def __error_on_required_status(self, required): 86 | """Error if tag is required.""" 87 | if required: 88 | err_prefix = "Watchmaker status tag required for azure resources," 89 | if not HAS_AZURE: 90 | err_msg = "required python sdk was not found" 91 | elif not self.resource_id or not self.subscription_id: 92 | err_msg = "resource and subcription ids \ 93 | were not found via metadata service" 94 | else: 95 | err_msg = "watchmaker was unable to update status" 96 | 97 | err_msg = f"{err_prefix} {err_msg}" 98 | self.logger.error(err_msg) 99 | raise StatusProviderError(err_msg) 100 | -------------------------------------------------------------------------------- /docs/troubleshooting/Windows/c_watchmaker_logs_watchmaker.log.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # The `c:\watchmaker\logs\watchmaker.log` Log-File 11 | 12 | This file tracks the top-level execution of the watchmaker configuration-utility. This file should _always_ exist. The primary reasons that it may not exist are: 13 | 14 | - The provisioning-administrator has checked for the log before the ``watchmaker``-utility has been downloaded and an execution-attempted. This typically happens if a ``watchmaker``-execution is attempted late in a complex provisioning-process 15 | 16 | - An execution-attempt wholly failed. In this case, check the logs for the watchmaker-calling service or process. 17 | 18 | - The provisioning-administrator has not invoked ``watchmaker`` in accordance with the ``watchmaker`` project's usage-guidance: if a different logging-location was specified (e.g., by adding a flag/argument like ``--log-dir=C:\TEMP\watchmaker``), the provisioning-administrator would need to check the alternately-specified logging-location. 19 | 20 | - The provisioning-administrator invoked the ``watchmaker``-managed content directly (e.g., using ``salt-call -c c:\watchmaker\salt\conf state.highstate``). In this scenario, only the content-execution may have been logged (whether logging was captured and where would depend on how the direct-execution was requested). 21 | 22 | ## Location Note 23 | 24 | The cited-location of the main ``watchmaker``-execution's log-file is predicated on the assumption that ``watchmaker`` has been executed per the Usage-guidance for [Windows](../../usage.md#windows): 25 | 26 | ```{eval-rst} 27 | .. literalinclude:: ../../usage.d/userData-Windows.ps1 28 | :language: shell 29 | :emphasize-lines: 21 30 | ``` 31 | 32 | The value of the ``--log-dir`` parameter sets the directory-location where ``watchmaker`` will create its log-files, including the ``watchmaker.log`` file. If a different value is set for the ``--log-dir`` parameter, the log-file will be created in _that_ directory-location, instead. 33 | 34 | 35 | ## Typical Errors 36 | 37 | * Bad specification of remotely-hosted configuration file. This will typically come with an HTTP 404 error similar to: 38 | ~~~ 39 | botocore.exceptions.ClientError: An error occurred (404) when calling the HeadObject operation: Not Found 40 | ~~~ 41 | Ensure that the requested URI for the remotely-hosted configuration file is valid. 42 | * Attempt to use a protected, remotely-hosted configuration-file. This will typically come win an HTTP 403 error. Most typically, this happens when the requested configuration-file exists on a protected network share and the requesting-process doesn't have permission to access it. 43 | ~~~ 44 | botocore.exceptions.ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden 45 | ~~~ 46 | Ensure that `watchmaker` has adequate permissions to access the requested, remotely-hosted configuration file. 47 | * Remotely-hosted configuration file is specified as an `s3://` URI without installation of `boto3` Python module. This will typically come with an error similar to: 48 | ```{eval-rst} 49 | .. literalinclude:: ../NoBoto3-LogSnippet.txt 50 | :language: text 51 | :emphasize-lines: 1-2 52 | ``` 53 | Ensure that the `boto3` Python module has been installed _prior to_ attempting to execute `watchmaker` 54 | 55 | ## Alternate Logs 56 | 57 | As noted above, this logfile may not exist if execution of watchmaker has wholly failed. If the execution was attempted via automated-startup methods but there is no watchmaker logfile, it will be necessary to check the CSP provider-logs. On AWS, the logs to check (per the [vendor documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html#ec2-windows-user-data)) will be: 58 | 59 | * If using (legacy) EC2Launch, the log-file to search will be [``C:\ProgramData\Amazon\EC2-Windows\Launch\Log\UserdataExecution.log``](c_amazon_EC2Launch_Log_UserdataExecution.log.md) 60 | * If using EC2Launch v2, the log-file to search will be [``C:\ProgramData\Amazon\EC2Launch\log\agent.log``](c_amazon_EC2Launch_v2_Log_UserdataExecution.log.md) 61 | -------------------------------------------------------------------------------- /src/watchmaker/status/providers/aws.py: -------------------------------------------------------------------------------- 1 | """AWS Status Provider.""" 2 | 3 | import logging 4 | 5 | from watchmaker import utils 6 | from watchmaker.conditions import HAS_BOTO3 7 | 8 | if HAS_BOTO3: 9 | import boto3 10 | 11 | from watchmaker.exceptions import StatusProviderError 12 | from watchmaker.status.providers.abstract import AbstractStatusProvider 13 | 14 | 15 | class AWSStatusProvider(AbstractStatusProvider): 16 | """Concrete implementation of the AWS status cloud provider.""" 17 | 18 | identifier = "aws" 19 | 20 | def __init__(self, provider, logger=None): 21 | self.logger = logger or logging.getLogger(__name__) 22 | self.metadata_id_url = "http://169.254.169.254/latest/meta-data/instance-id" 23 | self.metadata_region_url = ( 24 | "http://169.254.169.254/latest/meta-data/placement/region" 25 | ) 26 | 27 | self.instance_id = None 28 | self.region = None 29 | self.provider = provider 30 | self.initialize() 31 | 32 | def initialize(self): 33 | """Initialize instance id.""" 34 | if self.instance_id and self.region: 35 | return 36 | try: 37 | self.logger.debug("Initialize AWS instance_id and region") 38 | self.instance_id = self.__get_response_from_server(self.metadata_id_url) 39 | self.region = self.__get_response_from_server(self.metadata_region_url) 40 | except Exception: 41 | self.logger.exception("Error retrieving id/region from metadata service") 42 | 43 | def update_status(self, key, status, required): 44 | """Tag an AWS EC2 instance with the key and status provided.""" 45 | self.logger.debug("Tagging AWS Resource if HAS_BOTO3") 46 | self.logger.debug( 47 | "HAS_BOTO3=%s instance_id=%s status=%s", 48 | HAS_BOTO3, 49 | self.instance_id, 50 | status, 51 | ) 52 | if HAS_BOTO3 and self.instance_id and status: 53 | try: 54 | self.__tag_aws_instance(key, status) 55 | except Exception: 56 | self.logger.exception("Exception while tagging aws instance") 57 | else: 58 | return 59 | self.__error_on_required_status(required) 60 | 61 | def __tag_aws_instance(self, key, status): 62 | """Create or update instance tag with provided status.""" 63 | self.logger.debug("Tag Instance %s with %s:%s", self.instance_id, key, status) 64 | 65 | try: 66 | client = boto3.client("ec2", self.region) 67 | response = client.create_tags( 68 | Resources=[ 69 | self.instance_id, 70 | ], 71 | Tags=[ 72 | {"Key": key, "Value": status}, 73 | ], 74 | ) 75 | except Exception: 76 | self.logger.exception("Error tagging AWS instance") 77 | raise 78 | 79 | self.logger.debug("Create tag response %s", response) 80 | 81 | def __get_response_from_server(self, metadata_url): 82 | """Get response for provided metadata_url.""" 83 | headers = self.provider.get_metadata_request_headers() 84 | request = utils.urllib_utils.request.Request( 85 | metadata_url, 86 | data=None, 87 | headers=headers, 88 | ) 89 | response = utils.urlopen_retry( 90 | request, 91 | self.DEFAULT_TIMEOUT, 92 | ) 93 | return response.read().decode("utf-8") 94 | 95 | def __error_on_required_status(self, required): 96 | """Error if tag is required.""" 97 | if required: 98 | err_prefix = "Watchmaker status tag required for aws resources," 99 | if not HAS_BOTO3: 100 | err_msg = "required boto3 python sdk was not found" 101 | elif not self.instance_id: 102 | err_msg = "instance id was not found via metadata service" 103 | else: 104 | err_msg = "watchmaker was unable to update status" 105 | 106 | err_msg = f"{err_prefix} {err_msg}" 107 | self.logger.error(err_msg) 108 | raise StatusProviderError(err_msg) 109 | -------------------------------------------------------------------------------- /docs/customization/SiteFormulae.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # Modifying List of Executed-Formulas to Meet Site-needs 11 | 12 | The `watchmaker` utility bundles several SaltStack formulae. Which formulae are executed, in what order and under what conditions are governed by the `.../states/top.sls` file: 13 | 14 | - On Linux systems, the value of `.../` will be `/srv/watchmaker/salt` 15 | - On Windows systems, the value of `.../` will be `C:\Watchmaker\Salt\srv` 16 | 17 | A typical `.../states/top.sls` will look something like: 18 | 19 | ```{eval-rst} 20 | .. literalinclude:: FormulaTop-ALL.txt 21 | :language: shell 22 | ``` 23 | 24 | Adding, removing or re-ordering entries in this list modifies which formulae watchmaker executes and in what order it executes them 25 | 26 | ## Adding "Extra" Formulae to the Execution-List 27 | 28 | In order to add a new formula to Wachmaker's execution-list, edit the `.../states/top.sls` file. For cross-platform formulae, ensure appropriate entries exist under both the `base:G@os_family:RedHat` and `base:G@os_family:Winodws` lists. To add a formula to the execution list, insert the formula-name into the list just as the already-configured formulae are. For example, to add the [cribl-agent-formula] to the RedHat execution, modify the above RedHat stanza to look like: 29 | 30 | ```{eval-rst} 31 | .. literalinclude:: AddFormulaToTop-Simple-RedHat.txt 32 | :emphasize-lines: 7 33 | :language: yaml 34 | ``` 35 | 36 | If there are any futher conditionals that should be placed on the formula being added, surround the target-formula's list entry with suitable, Jinja-based conditional-operators. For example, if you want to ensure that the `cribl-agent` is executed when a suitable environment-value is specified, update the preceeding example to look like: 37 | 38 | ```{eval-rst} 39 | .. literalinclude:: AddFormulaToTop-Jinja-RedHat.txt 40 | :emphasize-lines: 7-9 41 | :language: shell 42 | ``` 43 | 44 | ## Removing Formulae the Execution-List 45 | 46 | In order to prevent a formula from being automatically run by Watchmaker, edit the `.../states/top.sls` file and either wholly remove the referenced-formula from the list or comment it out. To make the `scap-formula`'s `scan` state not run[^1], modify the example `.../states/top.sls` file to look like: 47 | 48 | ``` 49 | base: 50 | 'G@os_family:RedHat': 51 | - name-computer 52 | - scap.content 53 | - ash-linux.vendor 54 | - ash-linux.stig 55 | - ash-linux.iavm 56 | ``` 57 | 58 | or: 59 | 60 | ```{eval-rst} 61 | .. literalinclude:: RemoveFormulaFromTop-Simple-RedHat.txt 62 | :emphasize-lines: 7 63 | :language: yaml 64 | ``` 65 | 66 | ## Changing Formulae the Execution-Order 67 | 68 | There may be times where the system-owner will want Watchmaker to run formulae in a different order than previously-configured. The `.../states/top.sls` specifies formulae and states' execution-order serially. The order is top-to-bottom (with items closer to the top of the list executed earlier and those closer to the bottom of the lis executed later). To change the order that formulae are executed, change the order of the execution-list. 69 | 70 | ```{eval-rst} 71 | .. literalinclude:: FormulaTop-ALL_reordered.txt 72 | :emphasize-lines: 9,20 73 | :language: shell 74 | ``` 75 | 76 | In the above (when compared to the `.../states/top.sls` file near the top of this document), the Linux `scap.content` formula-state and the Windows `scap.scan` formula-state have been moved to a later executions. This is an atypical change, but is provided for completeness' sake. 77 | 78 | ```{eval-rst} 79 | .. note:: 80 | Some times, particularly when creating new states, it is dicsovered that 81 | some SaltStack formulae or states are not as idempotent as they were 82 | intended to be. Re-ordering the executions may work around issues caused by 83 | an insufficient degree of idempotency in one or more formulae. 84 | 85 | It is generally recommended, if idempotency-issues require execution-orders 86 | be modified, that the insufficiently-idempotent SaltStack formulae or states 87 | be refactored to improve their idempotency. 88 | ``` 89 | 90 | [^1]: This is often done by system-owners that value launch-time provisioning-automation speed over the presence of an intial hardening-scan report on the launched-systems hard drive. 91 | -------------------------------------------------------------------------------- /docs/gotchas/EL8-X11tunneling.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # X11-Forwarding Via SSH is "Broken" (EL8) 11 | 12 | The STIG-handlers for: 13 | 14 | * RHEL-08-040340 (and its Oracle equivalent, OL08-00-040340): 15 | > _RHEL 8 remote X connections for interactive users must be disabled unless to fulfill documented and validated mission requirements._ 16 | * RHEL-08-020041 (and its Oracle equivalent, OL08-00-020041): 17 | > _RHEL 8 must ensure session control is automatically started at shell initialization_ 18 | 19 | These settings will negatively-impact expected behaviors around X11-forwarding through SSH tunnels after application. 20 | 21 | ## Symptoms 22 | 23 | An error like "Can't open display" is emitted when attempting to launch X11 client-applications 24 | 25 | ## Things to Verify 26 | 27 | * Is `X11Forwarding` disabled in the `/etc/ssh/sshd_config` file 28 | * Is the `xauth` utility available: this utility is installed at `/usr/bin/xauth` through the installation of the `xorg-x11-xauth` RPM 29 | * Was X11-forwarding requested by the SSH client when establishing the connection to the remote host 30 | 31 | ## Fixes 32 | 33 | ### `X11Forwarding` is disabled in the `/etc/ssh/sshd_config` file 34 | 35 | Do the following to allow the use of X11-forwarding over SSH: 36 | 37 | 1. Execute `sudo grep -P '^(\s*)X11' /etc/ssh/sshd_config`. If the previous command returns null or shows a value of `no` 38 | 1. Update the `/etc/ssh/sshd_config` file to: 39 | * Ensure that any (uncommented) value for `X11Forwarding` is set to `yes` 40 | * Add a `X11Forwarding yes` line to the file if no uncommented `X11Forwarding` lines exist 41 | 1. Restart the `sshd` service 42 | 43 | ```{eval-rst} 44 | .. note:: 45 | 46 | The preceding will result in a scan-finding on any system that it has been 47 | executed on. The "…unless to fulfill documented and validated mission 48 | requirements" component of the STIG-rule means that provision of a 49 | documented reason for the enablement should allow the finding to be 50 | dismissed. 51 | ``` 52 | 53 | ### No `xauth` utility available 54 | 55 | * Ensure that the `xorg-x11-xauth` RPM is installed 56 | * Ensure that `/usr/bin` is in the user's `PATH` environment 57 | 58 | ### Ensure that X11 forwarding has been requested by your SSH client 59 | 60 | The methods for requesting X11 forwarding are specific to each SSH client. OpenSSH clients typically require including the `-Y` flag when requesting a connection. Consult vendor-documents for the proper setup of other SSH clients. 61 | 62 | ## Next Steps 63 | 64 | Assuming that all of the above verifications and associated fixes have been done and things still are not working as expected, it's likely that, when the login processes start up the `tmux` service, that the (tunneled) `DISPLAY` environment variable has not been properly set. This may occur because, when the `tmux` service is activated, the `DISPLAY` variable was not propagated from the initial login-shell to the `tmux`-managed subshell(s). If all other necessary components for setting up X-over-SSH are in place, the following should allow the session-user to set up an appropriate `DISPLAY` value within their `tmux` session-window: 65 | 66 | 1. Verify that your `${HOME}/.Xauthority` file exists 67 | 2. List out the authorization entries in the authority-file (execute `xauth list`). This should result in output like: 68 | ``` 69 | x-test.wam.lab/unix:10 MIT-MAGIC-COOKIE-1 dbb1c620b838faf7d5e5717d4a217a7c 70 | ``` 71 | On a freshly-launched system, there should be one and only one entry. If there's more than one entry, it means that either the file is stale or that more than one login-session has been concurrently opened to the remote host 72 | 3. Export the environment variable `DISPLAY`, setting it to `localhost:`. The value of `` will be the digit after the `/unix:` string in the `xauth list` output. Typically, this will mean executing `export DISPLAY=localhost:10`. 73 | 74 | Once the above has been done, attempts to launch X11 clients _should_ result in them displaying to the display of the system the operator has originally SSHed from. 75 | 76 | ```{eval-rst} 77 | .. note:: 78 | 79 | If one takes advantage of ``tmux``'s ability to multiplex terminal and one 80 | wishes to be able to launch X11 apps from any ``tmux``-managed session-window, 81 | it will be necessary to export the ``DISPLAY`` variable in *each* ``tmux`` 82 | session-window. 83 | ``` 84 | -------------------------------------------------------------------------------- /src/watchmaker/status/__init__.py: -------------------------------------------------------------------------------- 1 | """Status Module.""" 2 | 3 | import logging 4 | from typing import ClassVar 5 | 6 | import watchmaker.config.status as status_config 7 | from watchmaker.exceptions import CloudProviderDetectionError 8 | from watchmaker.status.providers.abstract import AbstractStatusProvider 9 | from watchmaker.status.providers.aws import AWSStatusProvider 10 | from watchmaker.status.providers.azure import AzureStatusProvider 11 | from watchmaker.utils.imds.detect import provider 12 | 13 | 14 | class Status: 15 | """Status factory for providers.""" 16 | 17 | _PROVIDERS: ClassVar[dict] = { 18 | AWSStatusProvider.identifier: AWSStatusProvider, 19 | AzureStatusProvider.identifier: AzureStatusProvider, 20 | } 21 | 22 | def __init__(self, config=None, logger=None): 23 | self.logger = logger or logging.getLogger(__name__) 24 | self.status_providers = {} 25 | self.providers = {} 26 | self.initialize(config) 27 | 28 | def initialize(self, config=None): 29 | """Initialize status providers.""" 30 | if not config: 31 | return 32 | 33 | detected_providers = self.__get_detected_providers(config) 34 | 35 | self.status_providers = self.__get_status_providers(detected_providers) 36 | 37 | for identifier in self.status_providers: 38 | self.providers[identifier] = status_config.get_providers_by_provider_types( 39 | config, 40 | identifier, 41 | ) 42 | 43 | def update_status(self, status): 44 | """Update status for each status provider.""" 45 | if not self.status_providers or not self.providers: 46 | return 47 | 48 | for identifier, providers in self.providers.items(): 49 | status_provider = self.status_providers.get(identifier) 50 | for provider_type in providers: 51 | status_provider.update_status( 52 | status_config.get_provider_key(provider_type), 53 | status_config.get_status(status), 54 | status_config.is_provider_required(provider_type), 55 | ) 56 | 57 | def get_detected_status_providers(self): 58 | """Get detected providers.""" 59 | return self.status_providers 60 | 61 | def __get_status_providers(self, detected_providers): 62 | """Get providers by identifiers.""" 63 | status_providers = {} 64 | for detected_provider in detected_providers: 65 | status_providers[detected_provider.identifier] = Status._PROVIDERS.get( 66 | detected_provider.identifier, 67 | )(detected_provider) 68 | 69 | return status_providers 70 | 71 | def __get_detected_providers(self, config): 72 | """Get detected status provider ids.""" 73 | detected_providers = [] 74 | 75 | detected_provider = self.__detect_provider_with_prereqs(config) 76 | 77 | if detected_provider and detected_provider.identifier: 78 | self.logger.debug("Detected provider %s", detected_provider.identifier) 79 | detected_providers.append(detected_provider) 80 | else: 81 | self.__error_on_required_provider(config) 82 | 83 | detected_providers.extend(status_config.get_non_cloud_providers(config)) 84 | 85 | return detected_providers 86 | 87 | def __detect_provider_with_prereqs(self, config): 88 | """Detect supported providers with prereqs.""" 89 | supported_providers = status_config.get_supported_cloud_w_prereqs(config) 90 | # Detect providers in config that have prereqs 91 | detected_provider = provider(supported_providers) 92 | return ( 93 | None 94 | if detected_provider.identifier == AbstractStatusProvider.identifier 95 | else detected_provider 96 | ) 97 | 98 | def __error_on_required_provider(self, config): 99 | """Detect required providers in config that do no have prereqs.""" 100 | req_providers_missing_prereqs = status_config.get_required_cloud_wo_prereqs( 101 | config, 102 | ) 103 | self.logger.debug( 104 | "For each required provider missing " 105 | "prereqs, attempt to detect provider: %s", 106 | req_providers_missing_prereqs, 107 | ) 108 | cloud_provider = provider(req_providers_missing_prereqs) 109 | 110 | # If a req provider is found raise StatusProviderError 111 | if cloud_provider.identifier != AbstractStatusProvider.identifier: 112 | raise CloudProviderDetectionError(cloud_provider.identifier) 113 | -------------------------------------------------------------------------------- /docs/customization/NewFormulas.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # Testing New Formulas 11 | 12 | The formulae-contents that are installed and configured for use by Watchmaker can be modified through a custom `config.yaml` file. This is done through the `config.yaml` file's `user_formula` dictionary-parameter (see: the [discussion](ConfigYaml.md) of the `config.yaml` file's `user_formulas` parameter and _take special note of guidance around file-formatting and indenting_). This parameter may be used to enable the setup of new, yet-to-be integrated formulae[^1]. This is done by specifying dictionary-values for `user_formulas` in a custom `config.yaml` file: 13 | 14 | ``` 15 | all: 16 | salt: 17 | [...elided...] 18 | user_formulas: 19 | : 20 | : 21 | [...elided...] 22 | : 23 | ``` 24 | 25 | ```{eval-rst} 26 | .. note:: 27 | While multiple new formulae are shown in the above snippet, it's not 28 | generally recommended to use this method for more than one, new formula at 29 | a time. The above is primarily to illustrate that the `user_formulas` 30 | parameter is a dictionary 31 | ``` 32 | 33 | Once the custom `config.yaml` file is in the desired state, it can be uploaded to an S3-based testing-bucket, web server[^2] or even staged locally within the testing-system. 34 | 35 | ## About Testing New Formulae 36 | 37 | To the greatest extent possible, formulae should be portable. It is recommended that when testing updates, the developer: 38 | 39 | * Tests without use of a custom `salt-content.zip` 40 | * Tests using custom Pillar-data – either by hand-modifying Pillar content or using a modified `salt-content.zip` cloned from the target deployment-environments' `salt-content.zip` – for one or more targeted deployment-environments 41 | 42 | Exercising across environments, in this way, will better assure that newly-created formulae operate as portably as expected prior to the newly-created formulae's integration into standard or site-specific Watchmaker executions. 43 | 44 | 45 | ## Execution - Generic/Defaults 46 | 47 | Assuming that the executing system has access to the specified URIs, watchmaker will: 48 | 49 | 1. Download the requested formula ZIP-archive(s) 50 | 2. Unarchive them to the `.../formulas` directory 51 | 3. Update the `.../minion` file's `file_roots:base` list 52 | 53 | If the site's `salt-content.zip` has not been modified to cause execution, the new formula can be explicitly executed using a method similar to: 54 | 55 | - Linux invocation: 56 | ```shell 57 | watchmaker \ 58 | -c s3:///config.yaml \ 59 | -s \ 60 | --log-level debug --log-dir=/var/log/watchmaker 61 | ``` 62 | - Windows invocation: 63 | ```shell 64 | watchmaker --log-level debug --log-dir=C:\Watchmaker\Logs -c s3:///config.yaml -s 65 | 66 | ``` 67 | The new formula's execution will be logged into the directory requested via the manual invocation. 68 | 69 | ## Execution - Tailored 70 | 71 | If the new formula has variable configuration-data that needs to come from pillar, it will be necessary to either manually update the `.../pillar` directory's contents with the appropiate data (see: [_The `pillar` Directory-Tree_](SaltContent.md#the-pillar-directory-tree)) or create a custom `salt-config.zip` file and reference it from the custom `config.yaml` file. 72 | 73 | ## Final Notes 74 | 75 | All formulae that have Pillar-settable or Pillar-overridable parameters should include a `pillar.example` or `pillar.example.yaml` file with the project's content. This file should be placed in the project's root-directory. The file should be valid YAML with explanatory comments for each configuration item. If a new formula's execution-customizability is more complex than is easily accommodated by comment-entries in the example Pillar YAML file, add a `README_PillarContents.md` file to the project. This file should contain sufficiently-expository content to allow new users of the formula to fully understand how to tailor the formulae's execution to their site's needs. 76 | 77 | [^1]: "Yet-to-be-integrated" formulae are any formulas that have not yet been set up for automated execution as part of a _full_ Watchmaker run. See the [_Modifying Formulae Execution-Parameters_](SiteParameters.md) document for tips. 78 | [^2]: If hosting on a web server and configuration content may be deemed sensitive, apply suitable access controls to the file and specify the fetch-URL with the appropriate authentication-elements. 79 | -------------------------------------------------------------------------------- /src/watchmaker/cli.py: -------------------------------------------------------------------------------- 1 | """Watchmaker cli.""" 2 | 3 | import os 4 | import platform 5 | import sys 6 | from pathlib import Path 7 | 8 | import click 9 | 10 | import watchmaker 11 | from watchmaker.logger import LOG_LEVELS, exception_hook, prepare_logging 12 | 13 | LOG_LOCATIONS = { 14 | "linux": str(Path("/var/log/watchmaker")), 15 | "windows": str( 16 | Path(os.environ.get("SYSTEMDRIVE", "C:") + "\\") / "Watchmaker" / "Logs", 17 | ), 18 | } 19 | 20 | 21 | def _print_version(ctx, _param, value): 22 | if not value or ctx.resilient_parsing: 23 | return 24 | click.echo(watchmaker.VERSION_INFO) 25 | ctx.exit() 26 | 27 | 28 | @click.command(context_settings={"ignore_unknown_options": True}) 29 | @click.option( 30 | "--version", 31 | is_flag=True, 32 | callback=_print_version, 33 | expose_value=False, 34 | is_eager=True, 35 | ) 36 | @click.option( 37 | "-c", 38 | "--config", 39 | "config_path", 40 | default=None, 41 | show_default=True, 42 | help=( 43 | "Path or URL to the config.yaml file. If not set, watchmaker will use " 44 | "its default config." 45 | ), 46 | ) 47 | @click.option( 48 | "-l", 49 | "--log-level", 50 | default="debug", 51 | show_default=True, 52 | type=click.Choice(list(LOG_LEVELS.keys())), 53 | help="Set the log level. Case-insensitive.", 54 | ) 55 | @click.option( 56 | "-d", 57 | "--log-dir", 58 | show_default=True, 59 | type=click.Path(exists=False, file_okay=False), 60 | default=LOG_LOCATIONS.get(platform.system().lower(), None), 61 | help=("Path to the directory where Watchmaker log files will be saved."), 62 | ) 63 | @click.option( 64 | "-n", 65 | "--no-reboot", 66 | "no_reboot", 67 | flag_value=True, 68 | default=False, 69 | show_default=True, 70 | help=( 71 | "If this flag is not passed, Watchmaker will reboot the " 72 | "system upon success. This flag suppresses that behavior. " 73 | "Watchmaker suppresses the reboot automatically if it " 74 | "encounters a failure." 75 | ), 76 | ) 77 | @click.option( 78 | "-s", 79 | "--salt-states", 80 | default=None, 81 | show_default=True, 82 | help=( 83 | "Comma-separated string of salt states to apply. A value of " 84 | "'none' will not apply any salt states. A value of " 85 | "'highstate' will apply the salt highstate." 86 | ), 87 | ) 88 | @click.option( 89 | "-A", 90 | "--admin-groups", 91 | default=None, 92 | show_default=True, 93 | help=( 94 | "Set a salt grain that specifies the domain groups that " 95 | "should have root privileges on Linux or admin privileges " 96 | "on Windows. Value must be a colon-separated string. E.g. " 97 | '"group1:group2"' 98 | ), 99 | ) 100 | @click.option( 101 | "-a", 102 | "--admin-users", 103 | default=None, 104 | show_default=True, 105 | help=( 106 | "Set a salt grain that specifies the domain users that " 107 | "should have root privileges on Linux or admin privileges " 108 | "on Windows. Value must be a colon-separated string. E.g. " 109 | '"user1:user2"' 110 | ), 111 | ) 112 | @click.option( 113 | "-t", 114 | "--computer-name", 115 | default=None, 116 | show_default=True, 117 | help=("Set a salt grain that specifies the computername to apply to the system."), 118 | ) 119 | @click.option( 120 | "-e", 121 | "--env", 122 | "environment", 123 | default=None, 124 | show_default=True, 125 | help=( 126 | "Set a salt grain that specifies the environment in which " 127 | "the system is being built. E.g. dev, test, or prod" 128 | ), 129 | ) 130 | @click.option( 131 | "-p", 132 | "--ou-path", 133 | default=None, 134 | show_default=True, 135 | help=( 136 | "Set a salt grain that specifies the full DN of the OU " 137 | "where the computer account will be created when joining a " 138 | 'domain. E.g. "OU=SuperCoolApp,DC=example,DC=com"' 139 | ), 140 | ) 141 | @click.argument("extra_arguments", nargs=-1, type=click.UNPROCESSED, metavar="") 142 | def main(extra_arguments=None, **kwargs): 143 | """Entry point for Watchmaker cli.""" 144 | # Convert log_dir to Path for the API 145 | if kwargs.get("log_dir"): 146 | kwargs["log_dir"] = Path(kwargs["log_dir"]) 147 | 148 | prepare_logging(kwargs["log_dir"], kwargs["log_level"]) 149 | 150 | sys.excepthook = exception_hook 151 | 152 | watchmaker_arguments = watchmaker.Arguments( 153 | extra_arguments=extra_arguments, 154 | **kwargs, 155 | ) 156 | watchmaker_client = watchmaker.Client(watchmaker_arguments) 157 | sys.exit(watchmaker_client.install()) 158 | -------------------------------------------------------------------------------- /src/watchmaker/config/status/__init__.py: -------------------------------------------------------------------------------- 1 | """Status Config module.""" 2 | 3 | import logging 4 | 5 | from watchmaker.conditions import HAS_AZURE, HAS_BOTO3 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | SUPPORTED_CLOUD_PROVIDERS = [ 10 | {"provider": "aws", "has_prereq": HAS_BOTO3}, 11 | {"provider": "azure", "has_prereq": HAS_AZURE}, 12 | ] 13 | SUPPORTED_NON_CLOUD_PROVIDERS = [] 14 | 15 | STATUS = { 16 | "RUNNING": "Running", 17 | "COMPLETE": "Completed", 18 | "ERROR": "Error", 19 | } 20 | 21 | 22 | def is_valid(config): 23 | """Validate config.""" 24 | if not config: 25 | return True 26 | 27 | providers = config.get("providers", None) 28 | if not providers: 29 | return False 30 | 31 | valid = True 32 | for provider in providers: 33 | if "key" not in provider or not provider["key"]: 34 | valid = False 35 | log.error("Status provider is missing key or value") 36 | if "provider_type" not in provider or not provider["provider_type"]: 37 | valid = False 38 | log.error("Status provider is missing provider_type or value") 39 | if not isinstance(provider.get("required"), bool): 40 | valid = False 41 | log.error("Status provider required value is not a bool") 42 | 43 | return valid 44 | 45 | 46 | def get_status(status_key): 47 | """ 48 | Get status message. 49 | 50 | returns string: formatted status message from key provided 51 | or status_key as status 52 | """ 53 | status = STATUS.get(status_key) 54 | return status if status else status_key 55 | 56 | 57 | def get_provider_key(provider): 58 | """Get key from the provider.""" 59 | return provider["key"] 60 | 61 | 62 | def is_provider_required(provider): 63 | """Get whether provider required.""" 64 | return provider.get("required", False) 65 | 66 | 67 | def get_provider_type(provider): 68 | """Get provider type.""" 69 | return provider["provider_type"] 70 | 71 | 72 | def get_providers_by_provider_types(config_status, provider_type): 73 | """Get the providers for the provider types.""" 74 | return [ 75 | provider 76 | for provider in config_status.get("providers", []) 77 | if provider["provider_type"].lower() == provider_type 78 | ] 79 | 80 | 81 | def get_supported_cloud_w_prereqs(config_status): 82 | """Get supported cloud providers with prereqs and in config.""" 83 | supported = get_cloud_with_prereqs() 84 | return list( 85 | { 86 | status.get("provider_type").lower() 87 | for status in config_status.get("providers", []) 88 | if status.get("provider_type", "").lower() in supported 89 | }, 90 | ) 91 | 92 | 93 | def get_required_cloud_wo_prereqs(config_status): 94 | """Get supported cloud providers without prereqs and in config.""" 95 | missing_prereqs = get_cloud_missing_prereqs() 96 | return list( 97 | { 98 | status.get("provider_type").lower() 99 | for status in config_status.get("providers", []) 100 | if status.get("provider_type", "").lower() in missing_prereqs 101 | and status.get("required", False) 102 | }, 103 | ) 104 | 105 | 106 | def get_cloud_with_prereqs(): 107 | """Get supported cloud providers with prereqs.""" 108 | providers = set() 109 | 110 | for cloud_provider in SUPPORTED_CLOUD_PROVIDERS: 111 | if cloud_provider["has_prereq"]: 112 | log.debug( 113 | "Adding %s to list of providers with prereqs", 114 | cloud_provider["provider"], 115 | ) 116 | providers.add(cloud_provider["provider"]) 117 | else: 118 | log.debug( 119 | "Skipping provider %s prereqs not found", 120 | cloud_provider["provider"], 121 | ) 122 | 123 | return list(providers) 124 | 125 | 126 | def get_cloud_missing_prereqs(): 127 | """Get supported cloud providers without prereqs.""" 128 | providers = set() 129 | 130 | for cloud_provider in SUPPORTED_CLOUD_PROVIDERS: 131 | if not cloud_provider["has_prereq"]: 132 | providers.add(cloud_provider["provider"]) 133 | 134 | return list(providers) 135 | 136 | 137 | def get_non_cloud_providers(config_status): 138 | """Get unique list of other provider providers.""" 139 | providers = set() 140 | for provider in SUPPORTED_NON_CLOUD_PROVIDERS: 141 | providers.add(provider["provider"]) 142 | 143 | non_cloud_provider_list = list(providers) 144 | 145 | return list( 146 | { 147 | status.get("provider_type").lower() 148 | for status in config_status.get("providers", []) 149 | if status.get("provider_type", "").lower() in non_cloud_provider_list 150 | }, 151 | ) 152 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | # Use uv's PEP 517 build backend for fast, reproducible builds 3 | build-backend = "uv_build" 4 | requires = ["uv-build<0.10"] 5 | 6 | [project] 7 | name = "watchmaker" 8 | version = "0.29.4" 9 | description = "Applied Configuration Management" 10 | readme = {file = "README.md", content-type = "text/markdown"} 11 | license = "Apache-2.0" 12 | license-files = ["LICENSE", "AUTHORS.md"] 13 | authors = [ 14 | {name = "Plus3IT Maintainers of Watchmaker", email = "projects@plus3it.com"}, 15 | ] 16 | requires-python = ">=3.8" 17 | classifiers = [ 18 | "Development Status :: 4 - Beta", 19 | "Intended Audience :: Developers", 20 | "Intended Audience :: System Administrators", 21 | "Operating System :: POSIX :: Linux", 22 | "Operating System :: Microsoft :: Windows", 23 | "Programming Language :: Python", 24 | "Programming Language :: Python :: 3", 25 | "Programming Language :: Python :: 3.8", 26 | "Programming Language :: Python :: 3.9", 27 | "Programming Language :: Python :: 3.10", 28 | "Programming Language :: Python :: 3.11", 29 | "Programming Language :: Python :: 3.12", 30 | "Programming Language :: Python :: Implementation :: CPython", 31 | "Programming Language :: Python :: Implementation :: PyPy", 32 | "Topic :: Utilities", 33 | ] 34 | dependencies = [ 35 | "backoff", 36 | "click", 37 | "defusedxml; platform_system=='Windows'", 38 | "distro", 39 | "importlib_resources; python_version<'3.9'", 40 | "pywin32; platform_system=='Windows'", 41 | "PyYAML", 42 | "compatibleversion>=0.1.2", 43 | "oschmod>=0.1.3", 44 | ] 45 | 46 | [project.urls] 47 | Homepage = "https://github.com/plus3it/watchmaker" 48 | Repository = "https://github.com/plus3it/watchmaker" 49 | Documentation = "https://watchmaker.cloudarmor.io" 50 | Changelog = "https://watchmaker.cloudarmor.io/en/stable/changelog.html" 51 | 52 | [project.scripts] 53 | wam = "watchmaker.cli:main" 54 | watchmaker = "watchmaker.cli:main" 55 | 56 | [tool.uv.build-backend] 57 | source-include = [ 58 | "CHANGELOG.md", 59 | "CONTRIBUTING.md", 60 | ] 61 | 62 | source-exclude = [ 63 | ".bumpversion.cfg", 64 | ".editorconfig", 65 | ".git", 66 | ".gitattributes", 67 | ".gitignore", 68 | ".gitkeep", 69 | ".github", 70 | ".travis.yml", 71 | ".mergify.yml", 72 | ".pydocstyle", 73 | ".pylintrc", 74 | ".yamllint.yml", 75 | "tests", 76 | ] 77 | 78 | [tool.ruff] 79 | # Line length and excludes chosen to match prior flake8 settings 80 | line-length = 88 81 | exclude = [ 82 | ".git", 83 | "__pycache__", 84 | ".eggs", 85 | "*.egg", 86 | "build", 87 | "dist", 88 | "htmlcov", 89 | "*/static/salt/formulas/*", 90 | ] 91 | 92 | [tool.ruff.lint] 93 | ignore = [ 94 | "B905", # zip() without strict= - requires Python 3.10+, project supports 3.8+ 95 | "D107", # __init__ docstrings are documented at the class level to avoid duplication 96 | "D203", # Conflicts with D211 (no-blank-line-before-class). Project uses D211 style. 97 | "D212", # Conflicts with D213 (multi-line-summary-first-line). Project uses D213 style. 98 | "S202", # tarfile.extractall without filter - requires Python 3.12+ or custom validation 99 | "EXE001", # Only occurs in CI when source is restored as artifact (loses executable bit) 100 | "ANN", # 500+ missing type annotations across src/tests; keep ignored for now; see staged plan below 101 | ] 102 | select = ["ALL"] 103 | 104 | [tool.ruff.lint.per-file-ignores] 105 | "tests/**" = ["S101", "S314", "SLF", "ARG", "PLR2004", "PLR0913", "INP001"] 106 | "docs/conf.py" = ["ERA001", "INP001"] 107 | "ci/**" = ["INP001"] 108 | "src/watchmaker/config/__init__.py" = ["PLR0915"] 109 | 110 | # Prepare for future gradual enabling of ANN (flake8-annotations) with saner defaults 111 | [tool.ruff.lint.flake8-annotations] 112 | # Don't require explicit "-> None" on __init__ (aligns with mypy default) 113 | mypy-init-return = true 114 | 115 | # Allow using Any for *args/**kwargs without error 116 | allow-star-arg-any = true 117 | 118 | # Don't flag unused conventional dummy args like _ or unused parameters in callbacks 119 | suppress-dummy-args = true 120 | 121 | # Don't require explicit -> None on functions that implicitly return None 122 | suppress-none-returning = true 123 | 124 | [tool.pytest.ini_options] 125 | # Limit discovery to dedicated test directory 126 | testpaths = ["tests"] 127 | 128 | # Only consider filenames starting with test_ 129 | python_files = ["test_*.py"] 130 | 131 | # Exclude large static datasets, build artifacts, generated hook dirs, and non-source roots 132 | norecursedirs = [ 133 | ".git", 134 | ".env", 135 | ".venv", 136 | "__pycache__", 137 | "ci/pyinstaller", 138 | "dist", 139 | "docs", 140 | "build", 141 | "htmlcov", 142 | "src/watchmaker/static/salt/formulas", 143 | ] 144 | 145 | # Additional pytest options: -r (show extra summary info for skips) and -a (all except passes) 146 | addopts = [ 147 | "-raxEfsw", 148 | "--strict-markers", 149 | "--doctest-modules", 150 | "--doctest-glob=*.md", 151 | "--tb=short", 152 | ] 153 | -------------------------------------------------------------------------------- /docs/troubleshooting/Linux/salt_call.debug.log.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # The `/var/log/watchmaker/salt_call.debug.log` Log-File 11 | 12 | This is the log-file that captures the bulk of the SaltStack-related state-output. This file gets created when `watchmaker` has been able to successfully download all of its execution information. This file gets created shortly after this line appears in the `/var/log/watchmaker/watchmaker.log` file: 13 | 14 | ~~~ 15 | 2023-06-15 11:13:27,378 [watchmaker.workers.base.SaltLinux][DEBUG][6407]: Command: /usr/bin/salt-call --local --retcode-passthrough --no-color --config-dir /opt/watchmaker/salt --log-file /var/log/watchmaker/salt_call.debug.log --log-file-level debug --log-level error --out quiet --return local state.highstate 16 | ~~~ 17 | 18 | Typically, the only errors that will appear here are the results of errors in the SaltStack formulae for the standard integrations. To see which modules _may_ get logged into this file, look at the contents of the `/srv/watchmaker/salt/formulas/` directory and then cross-reference those directories against the contents of the `/srv/watchmaker/salt/states/top.sls` file. To help interpret, a typical `top.sls` file's contents is offered: 19 | 20 | ~~~ 21 | 22 | {%- set environments = ['dev', 'test', 'prod', 'dx'] %} 23 | 24 | base: 25 | 'G@os_family:RedHat': 26 | - name-computer 27 | - scap.content 28 | - ash-linux.vendor 29 | - ash-linux.stig 30 | - ash-linux.iavm 31 | {%- if salt.grains.get('watchmaker:enterprise_environment') | lower in environments %} 32 | - join-domain 33 | - mcafee-agent 34 | - splunkforwarder 35 | - nessus-agent.elx.install 36 | # Recommend other custom states be inserted here 37 | {%- endif %} 38 | - scap.scan 39 | 40 | 'G@os_family:Windows': 41 | [...elided...] 42 | ~~~ 43 | 44 | In the above, these salt formulas will be executed unconditionally on RedHat-derivative systems: 45 | 46 | - `/srv/watchmaker/salt/formulas/name-computer-formula` 47 | - `/srv/watchmaker/salt/formulas/ash-linux-formula`[^1] 48 | 49 | Similarly, the contents of the following directories will be executed by `watchmaker` only if the environment specified in the `watchmaker`-invocation (the string-value after the `-e` flag) matches one of the elements in the `environments` list. 50 | 51 | - `/srv/watchmaker/salt/formulas/join-domain-formula` 52 | - `/srv/watchmaker/salt/formulas/mcafee-agent-formula` 53 | - `/srv/watchmaker/salt/formulas/nessus-agent-formula` 54 | - `/srv/watchmaker/salt/formulas/splunkforwarder-formula` 55 | 56 | Similarly, the behavior of each of the above states' executions will be governed by content specified under the `/srv/watchmaker/salt/pillar` directory hierarchy. This content is used to feed values into the parameter-driven SaltStack states enumerated in the `.../formulas` directories. 57 | 58 | ## Typical Error Causes 59 | 60 | The most frequent causes of errors, once `watchmaker` has caused `Saltstack` states to begin their execution, are errors encountered while running the individual enterprise-integration states. Typically, these errors are around stale configuration data (expired domain-join credentials for directory-integration or stale host/IP/port information for other services) or communication-issues between the OS that `watchmaker` is configuring and the service `watchmaker` is attempting to configure the instance to integrate: DNS resolution, host or network-level firewall rules, other transit-issues. 61 | 62 | The next most frequent errors are already-existing configuration problems in the OS that `watchmaker` is configuring. These include things like: 63 | - Failures accessing RPM repositories (especially problematic with repositories that require client-cert authentication where there are certificate-expiration problems between the RPM client and repository server) 64 | - Too little storage in critical partitions 65 | - The `watchmaker` activities running after something else has changed a resource-configuration that `watchmaker` expects to manage but finds the resource in an unanticipated state 66 | 67 | The least frequent cause of errors is related to the SaltStack code itself. Usually, this is caught in pre-release testing, but "bugs happen". While states are typically coded to try to gracefully handle errors encountered – they'll typically still fail, but at least try to provide meaningful error-output. Usually, the "bugs happen" errors are resultant of environment-to-environment deltas that were not adequately specified to the code-maintainers or the requisite logic-branching was not able to be adequately exercised across the various environments. 68 | 69 | For errors in enterprise-integration content, efforts have been undertaken to try to ensure those errors are adequately represented in this log-file. However, the application-specific logs (the ones _for_ the integrated-application) will still remain the authoritative source for troubleshooting exercises. 70 | 71 | [^1]: Due to the `ash-linux.vendor`, `ash-linux.stig` and `ash-linux.iavm` specification, only the `ash-linux-formula`'s `vendor`, `stig` and `iavm` states' executions will be attempted. 72 | 73 | -------------------------------------------------------------------------------- /src/watchmaker/workers/yum.py: -------------------------------------------------------------------------------- 1 | """Watchmaker yum worker.""" 2 | 3 | from pathlib import Path 4 | from typing import ClassVar 5 | 6 | import distro 7 | 8 | import watchmaker.utils 9 | from watchmaker.exceptions import WatchmakerError 10 | from watchmaker.managers.platform_manager import LinuxPlatformManager 11 | from watchmaker.workers.base import WorkerBase 12 | 13 | 14 | class Yum(WorkerBase, LinuxPlatformManager): 15 | """ 16 | Install yum repos. 17 | 18 | Args: 19 | repo_map: (:obj:`list`) 20 | List of dictionaries containing a map of yum repo files to systems. 21 | (*Default*: ``[]``) 22 | 23 | """ 24 | 25 | SUPPORTED_DISTS: ClassVar[dict] = { 26 | "almalinux": "almalinux", 27 | "amazon": "amazon", 28 | "amzn": "al2023", 29 | "centos": "centos", 30 | "oracle": "oracle", 31 | "rhel": "redhat", 32 | "rocky": "rocky", 33 | } 34 | 35 | def __init__(self, *args, **kwargs): 36 | # Pop arguments used by Yum 37 | self.yumrepomap = kwargs.pop("repo_map", None) or [] 38 | 39 | # Init inherited classes 40 | super().__init__(*args, **kwargs) 41 | self.dist_info = self.get_dist_info() 42 | 43 | def _get_amazon_el_version(self, version): 44 | # All amzn linux distros currently available use el6-based packages. 45 | # When/if amzn linux switches a distro to el7, rethink this. 46 | self.log.debug("Amazon Linux, version=%s", version) 47 | return "6" 48 | 49 | def get_dist_info(self): 50 | """Validate the Linux distro and return info about the distribution.""" 51 | dist = self.get_mapped_dist_name() 52 | version = distro.version().split(".")[0] 53 | el_version = None 54 | 55 | # Determine el_version 56 | if dist == "amazon": 57 | el_version = self._get_amazon_el_version(version) 58 | else: 59 | el_version = distro.version().split(".")[0] 60 | 61 | if el_version is None: 62 | msg = f"Unsupported OS version! dist = {dist}, version = {version}." 63 | self.log.critical(msg) 64 | raise WatchmakerError(msg) 65 | 66 | dist_info = {"dist": dist, "el_version": el_version} 67 | self.log.debug("dist_info=%s", dist_info) 68 | return dist_info 69 | 70 | def get_mapped_dist_name(self): 71 | """Return a normalized dist-name value.""" 72 | # Error if 'dist' is not found in SUPPORTED_DISTS 73 | try: 74 | return self.SUPPORTED_DISTS[distro.id()] 75 | except KeyError as exc: 76 | # Release not supported, exit with error 77 | msg = "Unsupported OS distribution. OS must be one of: {}".format( 78 | ", ".join(self.SUPPORTED_DISTS.keys()), 79 | ) 80 | self.log.critical(msg) 81 | raise WatchmakerError(msg) from exc 82 | 83 | def _validate_config(self): 84 | """Validate the config is properly formed.""" 85 | if not self.yumrepomap: 86 | self.log.warning("`yumrepomap` did not exist or was empty.") 87 | elif not isinstance(self.yumrepomap, list): 88 | msg = "`yumrepomap` must be a list!" 89 | self.log.critical(msg) 90 | raise WatchmakerError(msg) 91 | 92 | def _validate_repo(self, repo): 93 | """Check if a repo is applicable to this system.""" 94 | # Check if this repo applies to this system's dist and el_version. 95 | # repo['dist'] must match this system's dist or the keyword 'all' 96 | # repo['el_version'] is optional, but if present then it must match 97 | # this system's el_version. 98 | dist = self.dist_info["dist"] 99 | el_version = self.dist_info["el_version"] 100 | 101 | repo_dists = repo["dist"] 102 | if isinstance(repo_dists, str): 103 | # ensure repo_dist is a list 104 | repo_dists = [repo_dists] 105 | 106 | # is repo dist applicable to this system? 107 | check_dist = bool(set(repo_dists).intersection([dist, "all"])) 108 | 109 | # is repo el_version applicable to this system? 110 | check_el_version = "el_version" in repo and str(repo["el_version"]) == str( 111 | el_version, 112 | ) 113 | 114 | # return True if all checks pass, otherwise False 115 | return check_dist and check_el_version 116 | 117 | def before_install(self): 118 | """Validate configuration before starting install.""" 119 | 120 | def install(self): 121 | """Install yum repos defined in config file.""" 122 | self._validate_config() 123 | 124 | for repo in self.yumrepomap: 125 | if self._validate_repo(repo): 126 | # Download the yum repo definition to /etc/yum.repos.d/ 127 | self.log.info("Installing repo: %s", repo["url"]) 128 | url = repo["url"] 129 | repofile = Path( 130 | "/etc/yum.repos.d", 131 | ) / watchmaker.utils.basename_from_uri(url) 132 | self.retrieve_file(url, repofile) 133 | else: 134 | self.log.debug( 135 | "Skipped repo because it is not valid for this system: " 136 | "dist_info=%s", 137 | self.dist_info, 138 | ) 139 | self.log.debug("Skipped repo=%s", repo) 140 | -------------------------------------------------------------------------------- /docs/troubleshooting/Linux/journald.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. image:: /images/cropped-plus3it-logo-cmyk.png 4 | :width: 140px 5 | :alt: Powered by Plus3 IT Systems 6 | :align: right 7 | :target: https://www.plus3it.com 8 | 9 | ============== 10 | Using Journald 11 | ============== 12 | 13 | Each of the relevant, boot-initiated processes may be viewed with the ``journalctl`` utility. This utility can provide wholly-unfiltered output as well as allowing the operator to drill-down into more-specific logged data. 14 | 15 | ---------------------- 16 | View *All* Logged Data 17 | ---------------------- 18 | 19 | The most basic method for viewing data stored in ``journald`` is to simply execute ``journalctl`` with no arguments. This will display a paginated-view of all logged data. This paginated view has the side-effect of truncating the logged-output to the terminal's width. To avoid pagination/truncation and, instead, see line-wrapped lines, it will be necessary to add the ``--no-pager`` flag. Since this defeats pagination, it will typically also be desirable to pipe the whole output-stream through a tool like ``more`` or ``less`` (if using ``less``, include the flag ``-r`` to preserve ANSI formatting-code interpretation). 20 | 21 | To sum up, use: ``journalctl --no-pager | less -r`` 22 | 23 | ------------------------------------ 24 | View all data logged since last boot 25 | ------------------------------------ 26 | 27 | If using an AMI or VM-template such as those created with `spel `_, the journald service will have been configured for persistent-logging. As such, the above ``journalctl`` information will display log-information for every system-epoch. To restrict the view to *only* the most recent boot, add the flag/option, ``-b 0``. Similarly, to view only the logged data from before the current boot-epoch, add the flag/option, ``-b 1``. 28 | 29 | To sum up, use ``journalctl -b 0 --no-pager | less -r`` 30 | 31 | ---------------------------------- 32 | Viewing *Only* `cloud-init` Output 33 | ---------------------------------- 34 | 35 | The ``journalctl`` utility allows you to filter output via a number of means. One of those methods is via systemd service-unit name. Since ``cloud-init`` is a systemd service-unit, the ``journald`` output may be restricted to *only* things logged through *that* service. To do so, execute ``journalctl -u cloud-init`` 36 | 37 | ------------------------------ 38 | Viewing userData script-output 39 | ------------------------------ 40 | 41 | Applying a filter to show only userData script-output requires a couple of things: 42 | 43 | 1. Having enabled userData script(s) to emit logging information (e.g., through the ``logger`` utility) 44 | 2. Knowing what name the logging-enabled userData script(s) has been configured to report under 45 | 46 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 47 | Configuring userData Script(s) for Log-Capture 48 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 49 | 50 | Absent explicit setup, userData scripts won't have been configured to emit logging-information in a way that the `journald` service will have captured. A simple wat to set up such logging is to add a line similar to: 51 | 52 | .. code-block:: bash 53 | 54 | exec 1> >( logger -s -t "$( basename "${0}" )" ) 2>&1 55 | 56 | 57 | Near the top of the target userData script. This ensures that all STDOUT and STDERR content is sent to `journald` via the `logger` utility. The above is a BASH-ism. This won't work for POSIX-mode scripts and scripts using other interpreters will typically require their own syntax. 58 | 59 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 60 | Filtering for Logging-Enabled userData Script(s) Output 61 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 62 | 63 | If using the above, ``logger``-based method, output may be filtered either via the service-name, the logged-events' ``COMM``-attribute or the userData script's name (``journalctl SYSLOG_IDENTIFIER=``): 64 | 65 | - Using service-name: In this case, the service-name is ``cloud-final.service``. This means, one would use ``journalctl -u cloud-final`` 66 | - Using the logged-event's program-name: This is done using the ``_COMM`` flag. For a ``logger-enabled`` script, the name to filter on would be ``logger``. This means one would use ``journalctl _COMM=logger`` 67 | - Using the userData script's name by using ``journalctl SYSLOG_IDENTIFIER=``. Depending on how the userData script-payload had been declared, this will either be ``part-001`` or a specifically-requested script-name. To specifically request a name for the userData script(s), it will be necessary to have either declared a logging name other than ``"$( basename "${0}" )"`` or to have encapsulated the script in a MIME-stream like: 68 | 69 | .. code-block:: bash 70 | 71 | --===============BOUNDARY== 72 | Content-Disposition: attachment; filename="userData-script.sh" 73 | Content-Transfer-Encoding: 7bit 74 | Content-Type: text/x-shellscript 75 | Mime-Version: 1.0 76 | 77 | #!/bin/bash 78 | set -x 79 | # 80 | # UserData script 81 | ################################################################# 82 | 83 | # Log everything below into syslog 84 | exec 1> >( logger -s -t "$( basename "${0}" )" ) 2>&1 85 | 86 | 87 | 88 | ... 89 | 90 | 91 | The ``SCRIPT_SHORT_NAME`` value will be the same as the value of the "``attachment``'s" ``filename`` argument (in the above case, ``userData-script.sh``). 92 | -------------------------------------------------------------------------------- /docs/customization/SiteParameters.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # Modifying Formulae Execution-Parameters 11 | 12 | The `watchmaker` utility bundles several SaltStack formulae. These bundled-formulae's behaviors are, in turn, governed, by a set of [Pillar](https://docs.saltproject.io/salt/user-guide/en/latest/topics/pillar.html)-data that are also bundled with the `watchmaker`utility: pillar-data is how SaltStack states' behaviors may be modified. Sites that wish to either override the bundled-formulae's default behaviors or wish to run additional SaltStack formulae that are _not_ in the default formula-bundle – and need to provide supporting behvior-tailoring – can do so by creating custom pillar-data. 13 | 14 | ## Reference 15 | 16 | When customizing Pillar content, it will also be necessary to use a site-specific, YAML-formatted configuration-file. Per the "[Usage](../usage.md)" document's "from the CLI" section, this file is _typically_ named `config.yaml`. Any file name can be used so long as it matches what is passed via the `-c`/`--config` argument to the `watchmaker` utility. Further, this configuration-file may be specified as hosted on the local filesystem, any HTTP/HTTPS URL or an S3-hosted URI. 17 | 18 | The `watchmaker` pillar-data is delivered by way of a ZIP-formatted content-archive. While this archive-file typically takes the name `salt-content.zip`, any filename may be used so long as it's properly referenced in the `watchmaker` configuration-file's `salt_content` directive (see the [`config.yaml` discussion](ConfigYaml.md) for a deeper dive into this file's contents, including a discussion of the `salt_content` parameter). The following exmaple configuration-file – with `salt_content` directive highlighted – is taken from [watchmaker project](https://github.com/plus3it/watchmaker/blob/main/src/watchmaker/static/config.yaml): 19 | 20 | ```{eval-rst} 21 | .. literalinclude:: Example-config.yaml 22 | :emphasize-lines: 10 23 | :language: yaml 24 | .. note:: If creating a new config-file for customizing your site's ``watchmaker``-execution, it's recommended that config-file content *not* be copied from this document but from the ``watchmaker`` project, directly. 25 | ``` 26 | 27 | As with the configuration-file (passed via the `-c`/`--config` argument to the `watchmaker` utility), this file may be specified as hosted on the local filesystem, any HTTP/HTTPS URL or an S3-hosted URI. 28 | 29 | ## Site-Specific Parameters 30 | 31 | To implement localized-behavior with watchmaker, it will be necessary to change the `salt_content` paramter's value from `null` to the location of a SaltStack content-bundle. As mentioned previously, the content-bundle should be delivered in the form of a ZIP-formatted content-archive. 32 | 33 | The the overall structure and format of the archive-bundle is discussed in the [_Salt Contents Archive File_](SaltContent.md) document. Site-specific _parameters_ – and associated values – would be handled within the archive-bundle's [Pillar](SaltContent.md#the-pillar-directory-tree) data contents. 34 | 35 | ## Bundle locations 36 | 37 | Watchmaker currently supports pulling the Saltstack content-bundles from three types of locations: HTTP(S) server, S3 bucket or filesystem-path. The `salt_content` paramter's value is stated as a URI-notation. See the following subsections for guidance on location-specification. 38 | 39 | It's worth noting that Watchmaker has not been tested to (directly) support accessing CIFS- or NFS-based network-shares. If it is desired to access a content-bundle from such a hosting-location, it is recommended to include share-mounting steps in any pre-Watchmaker execution-steps. Once the network-share is mounted, then watchmaker can access the content-bundle as though it was a locally-staged bundle (see below). 40 | 41 | ### S3-Hosted Bundle 42 | 43 | An S3-hosted bundle would be specified like: 44 | 45 | ``` 46 | s3://// 47 | ``` 48 | 49 | For example, "`s3://my-site-bukkit/watchmaker/salt-content.zip`" 50 | 51 | ```{eval-rst} 52 | .. note:: 53 | For S3-hosted URIs, it will be necessary to have ensured that the 54 | Python Boto3 modules have been installed prior to executing watchmaker 55 | ``` 56 | 57 | ### Webserver-Hosted Bundle 58 | 59 | A bundle hosted on an HTTP server would be specified like: 60 | 61 | ``` 62 | https://// 63 | ``` 64 | 65 | For example, "`https://wamstuff.my-site.local/watchmaker/salt-content.zip`" 66 | 67 | ```{eval-rst} 68 | .. note:: 69 | Either HTTP or TLS-encrypted HTTP URIs are supported. 70 | 71 | If potentially-sensitive data will be contained in the site-localization 72 | archive-file, it is recommended that access to this file be restricted. 73 | This can typically be done with authorized IP-blocks, API tokens or other 74 | "simple" authentication credentials. If this limitation comes in the form of 75 | an API token or a simple-auth credential, it will be necessary to specify 76 | the token or credentials as part of the HTTP URI. 77 | ``` 78 | 79 | ### Locally-staged Bundle 80 | 81 | A locally-staged bundle (presumably downloaded and placed as part of a previously-executed launch-time automation-task) would be specified like: 82 | 83 | ``` 84 | file://// 85 | ``` 86 | 87 | For example, "`file:///var/tmp/watchmaker/salt-content.zip`" 88 | 89 | -------------------------------------------------------------------------------- /tests/test_watchmaker.py: -------------------------------------------------------------------------------- 1 | """Watchmaker main test module.""" 2 | 3 | import platform 4 | from pathlib import Path 5 | from unittest.mock import patch 6 | 7 | import pytest 8 | 9 | import watchmaker 10 | 11 | 12 | def test_main(): 13 | """Placeholder for tests.""" 14 | assert watchmaker.__version__ == watchmaker.__version__ 15 | 16 | 17 | def test_none_arguments(): 18 | """Check string 'None' conversion to None.""" 19 | raw_arguments = { 20 | "admin_groups": "None", 21 | "admin_users": "None", 22 | "computer_name": "None", 23 | "salt_states": "None", 24 | "ou_path": "None", 25 | } 26 | watchmaker_arguments = watchmaker.Arguments(**raw_arguments) 27 | 28 | assert watchmaker_arguments.admin_groups is None 29 | assert watchmaker_arguments.admin_users is None 30 | assert watchmaker_arguments.computer_name is None 31 | assert watchmaker_arguments.salt_states is None 32 | assert watchmaker_arguments.ou_path is None 33 | 34 | with patch("watchmaker.status.Status.initialize", return_value=None): 35 | watchmaker_client = watchmaker.Client(watchmaker_arguments) 36 | 37 | assert "salt_states" in watchmaker_client.worker_args 38 | assert watchmaker_client.worker_args["salt_states"] is None 39 | 40 | 41 | def test_argument_default_value(): 42 | """Ensure argument default value is `Arguments.DEFAULT_VALUE`.""" 43 | raw_arguments = {} 44 | check_val = watchmaker.Arguments.DEFAULT_VALUE 45 | watchmaker_arguments = watchmaker.Arguments(**raw_arguments) 46 | 47 | assert watchmaker_arguments.admin_groups == check_val 48 | assert watchmaker_arguments.admin_users == check_val 49 | assert watchmaker_arguments.computer_name == check_val 50 | assert watchmaker_arguments.salt_states == check_val 51 | assert watchmaker_arguments.ou_path == check_val 52 | 53 | 54 | def test_extra_arguments_string(): 55 | """Test string in extra_arguments loads correctly.""" 56 | # setup 57 | raw_arguments = {"extra_arguments": ["--foo", "bar"]} 58 | check_val = {"foo": "bar"} 59 | watchmaker_arguments = watchmaker.Arguments(**raw_arguments) 60 | 61 | # test 62 | with patch("watchmaker.status.Status.initialize", return_value=None): 63 | watchmaker_client = watchmaker.Client(watchmaker_arguments) 64 | 65 | # assertions 66 | assert watchmaker_client.worker_args == check_val 67 | 68 | 69 | def test_extra_arguments_equal_separator(): 70 | """Test equal separator in extra_arguments loads correctly.""" 71 | # setup 72 | raw_arguments = { 73 | "extra_arguments": [ 74 | "--foo=bar", 75 | ], 76 | } 77 | check_val = {"foo": "bar"} 78 | watchmaker_arguments = watchmaker.Arguments(**raw_arguments) 79 | 80 | # test 81 | with patch("watchmaker.status.Status.initialize", return_value=None): 82 | watchmaker_client = watchmaker.Client(watchmaker_arguments) 83 | 84 | # assertions 85 | assert watchmaker_client.worker_args == check_val 86 | 87 | 88 | def test_extra_arguments_quoted_string(): 89 | """Test quoted string in extra_arguments loads correctly.""" 90 | # setup 91 | raw_arguments = {"extra_arguments": ["--foo", '"bar"']} 92 | check_val = {"foo": "bar"} 93 | watchmaker_arguments = watchmaker.Arguments(**raw_arguments) 94 | 95 | # test 96 | with patch("watchmaker.status.Status.initialize", return_value=None): 97 | watchmaker_client = watchmaker.Client(watchmaker_arguments) 98 | 99 | # assertions 100 | assert watchmaker_client.worker_args == check_val 101 | 102 | 103 | def test_extra_arguments_list(): 104 | """Test list in extra_arguments loads correctly.""" 105 | # setup 106 | raw_arguments = {"extra_arguments": ["--foo", '["bar"]']} 107 | check_val = {"foo": ["bar"]} 108 | watchmaker_arguments = watchmaker.Arguments(**raw_arguments) 109 | 110 | # test 111 | with patch("watchmaker.status.Status.initialize", return_value=None): 112 | watchmaker_client = watchmaker.Client(watchmaker_arguments) 113 | 114 | # assertions 115 | assert watchmaker_client.worker_args == check_val 116 | 117 | 118 | def test_extra_arguments_map(): 119 | """Test map in extra_arguments loads correctly.""" 120 | # setup 121 | raw_arguments = { 122 | "extra_arguments": [ 123 | "--user-formulas", 124 | '{"foo-formula": "https://url"}', 125 | ], 126 | } 127 | check_val = {"user_formulas": {"foo-formula": "https://url"}} 128 | watchmaker_arguments = watchmaker.Arguments(**raw_arguments) 129 | 130 | # test 131 | with patch("watchmaker.status.Status.initialize", return_value=None): 132 | watchmaker_client = watchmaker.Client(watchmaker_arguments) 133 | 134 | # assertions 135 | assert watchmaker_client.worker_args == check_val 136 | 137 | 138 | @pytest.mark.skipif( 139 | platform.system() != "Windows", 140 | reason="Windows prepdir test only applies on Windows", 141 | ) 142 | def test_windows_prepdir_default(): 143 | r"""Test that Windows prepdir resolves to C:\Watchmaker.""" 144 | # setup 145 | raw_arguments = {} 146 | watchmaker_arguments = watchmaker.Arguments(**raw_arguments) 147 | 148 | # test 149 | with patch("watchmaker.status.Status.initialize", return_value=None): 150 | watchmaker_client = watchmaker.Client(watchmaker_arguments) 151 | 152 | # assertions 153 | expected_prepdir = Path("C:\\Watchmaker") 154 | assert watchmaker_client.system_params["prepdir"] == expected_prepdir 155 | -------------------------------------------------------------------------------- /ci/build_pyapp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu -o pipefail 3 | 4 | PYTHON=3.12 5 | 6 | VERSION=$(grep -E '^version\s*=' pyproject.toml | sed 's/^version = "\(.*\)"$/\1/') 7 | PYAPP_VERSION=$(sed -n 's/.*ofek\/pyapp@v//p' .github/workflows/dependabot_hack.yml) 8 | PYTHON_BUILD_STANDALONE_VERSION=$(sed -n 's/.*python-build-standalone@//p' .github/workflows/dependabot_hack.yml) 9 | 10 | PYAPP_DIST_DIR=".pyapp/dist/${VERSION}" 11 | PYAPP_BUILD_DIR=".pyapp/build" 12 | WAM_FILENAME="watchmaker-${VERSION}-standalone-linux-x86_64" 13 | 14 | # Python standalone build info - retrieve the latest patch version for Python 3.12 15 | echo "Fetching Python ${PYTHON} version from python-build-standalone release ${PYTHON_BUILD_STANDALONE_VERSION}..." 16 | PYTHON_RELEASE_URL="https://github.com/astral-sh/python-build-standalone/releases/expanded_assets/${PYTHON_BUILD_STANDALONE_VERSION}" 17 | PYTHON_FULL_VERSION=$(curl -sSL "$PYTHON_RELEASE_URL" | grep -oP "cpython-\K${PYTHON}\.\d+(?=\+${PYTHON_BUILD_STANDALONE_VERSION}-x86_64-unknown-linux-gnu-install_only_stripped\.tar\.gz)" | head -n 1) 18 | 19 | if [ -z "$PYTHON_FULL_VERSION" ]; then 20 | echo "Error: Could not determine Python ${PYTHON} patch version from release ${PYTHON_BUILD_STANDALONE_VERSION}" 21 | exit 1 22 | fi 23 | 24 | PYTHON_RELEASE="cpython-${PYTHON_FULL_VERSION}+${PYTHON_BUILD_STANDALONE_VERSION}-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" 25 | PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/${PYTHON_BUILD_STANDALONE_VERSION}/${PYTHON_RELEASE}" 26 | 27 | echo "Building PyApp standalone for watchmaker v${VERSION}..." 28 | echo "Using PyApp v${PYAPP_VERSION}" 29 | echo "Using Python ${PYTHON_FULL_VERSION} from python-build-standalone" 30 | 31 | echo "-----------------------------------------------------------------------" 32 | cargo --version 33 | rustc --version 34 | echo "-----------------------------------------------------------------------" 35 | 36 | # Build wheel if dist directory doesn't exist 37 | if [ ! -d "dist" ]; then 38 | echo "dist directory not found, building wheel..." 39 | uv build --wheel 40 | fi 41 | 42 | # Find the wheel file 43 | WHEEL_FILE=$(find dist -name "watchmaker-${VERSION}-py3-none-any.whl" | head -n 1) 44 | if [ -z "$WHEEL_FILE" ]; then 45 | echo "Error: Could not find wheel file for version ${VERSION}" 46 | exit 1 47 | fi 48 | WHEEL_FILE_ABS=$(realpath "$WHEEL_FILE") 49 | echo "Using wheel: $WHEEL_FILE_ABS" 50 | 51 | # Download and prepare custom Python distribution 52 | echo "Downloading Python standalone distribution..." 53 | mkdir -p "$PYAPP_BUILD_DIR" 54 | cd "$PYAPP_BUILD_DIR" 55 | 56 | if [ ! -f "$PYTHON_RELEASE" ]; then 57 | curl -L -o "$PYTHON_RELEASE" "$PYTHON_URL" 58 | fi 59 | 60 | PYTHON_DIR="python" 61 | 62 | # Remove any existing python directory to avoid overlapping files 63 | if [ -d "$PYTHON_DIR" ]; then 64 | rm -rf "$PYTHON_DIR" 65 | fi 66 | 67 | echo "Extracting Python distribution..." 68 | tar -xzf "$PYTHON_RELEASE" 69 | 70 | PYTHON_BIN="${PYTHON_DIR}/bin/python${PYTHON}" 71 | 72 | echo "Installing watchmaker and dependencies into custom Python distribution..." 73 | export UV_CACHE_DIR="${HOME}/.local/share/uv" 74 | uv pip install --python "$PYTHON_BIN" "$WHEEL_FILE_ABS" boto3 75 | 76 | echo "Creating custom distribution archive..." 77 | CUSTOM_DIST="cpython-${PYTHON_FULL_VERSION}-watchmaker-${VERSION}.tar.zst" 78 | tar -I zstd -cf "$CUSTOM_DIST" "$PYTHON_DIR" 79 | 80 | echo "Custom distribution created: $CUSTOM_DIST" 81 | ls -lh "$CUSTOM_DIST" 82 | 83 | # Build the standalone with PyApp via cargo install 84 | echo "Building PyApp standalone with cargo install..." 85 | cd ../.. 86 | mkdir -p "$PYAPP_DIST_DIR" 87 | 88 | export PYAPP_PROJECT_NAME="watchmaker" 89 | export PYAPP_PROJECT_VERSION="$VERSION" 90 | export PYAPP_DISTRIBUTION_EMBED=1 91 | PYAPP_DISTRIBUTION_PATH="$(realpath "${PYAPP_BUILD_DIR}/${CUSTOM_DIST}")" 92 | export PYAPP_DISTRIBUTION_PATH 93 | export PYAPP_DISTRIBUTION_PYTHON_PATH="python/bin/python${PYTHON}" 94 | export PYAPP_FULL_ISOLATION=1 95 | export PYAPP_SKIP_INSTALL=1 96 | 97 | # Use a persistent target directory for build cache 98 | export CARGO_TARGET_DIR="${PYAPP_BUILD_DIR}/target" 99 | PYAPP_INSTALL_DIR="${PYAPP_BUILD_DIR}/install" 100 | 101 | cargo install pyapp --locked --force --version "${PYAPP_VERSION}" --root "${PYAPP_INSTALL_DIR}" --target-dir "${CARGO_TARGET_DIR}" 102 | 103 | # Move the binary from cargo install location to final location 104 | if [ "${PYAPP_INSTALL_DIR}/bin/pyapp" -ef "${PYAPP_DIST_DIR}/${WAM_FILENAME}" ]; then 105 | rm -f "${PYAPP_DIST_DIR}/${WAM_FILENAME}" 106 | fi 107 | mv "${PYAPP_INSTALL_DIR}/bin/pyapp" "${PYAPP_DIST_DIR}/${WAM_FILENAME}" 108 | 109 | echo "Creating sha256 hashes of standalone binary..." 110 | (cd "$PYAPP_DIST_DIR"; sha256sum "$WAM_FILENAME" > "${WAM_FILENAME}.sha256") 111 | cat "${PYAPP_DIST_DIR}/${WAM_FILENAME}.sha256" 112 | 113 | echo "Setting executable permissions..." 114 | chmod +x "${PYAPP_DIST_DIR}/${WAM_FILENAME}" 115 | 116 | echo "Checking standalone binary version..." 117 | VERSION_OUTPUT=$("${PYAPP_DIST_DIR}/${WAM_FILENAME}" --version) 118 | echo "$VERSION_OUTPUT" 119 | 120 | echo "Validating versions..." 121 | if ! echo "$VERSION_OUTPUT" | grep -q "Watchmaker/${VERSION}"; then 122 | echo "Error: Expected Watchmaker version ${VERSION}, but got: $VERSION_OUTPUT" 123 | exit 1 124 | fi 125 | 126 | if ! echo "$VERSION_OUTPUT" | grep -q "Python/${PYTHON_FULL_VERSION}"; then 127 | echo "Error: Expected Python version ${PYTHON_FULL_VERSION}, but got: $VERSION_OUTPUT" 128 | exit 1 129 | fi 130 | 131 | echo "Version validation successful: Watchmaker ${VERSION} with Python ${PYTHON_FULL_VERSION}" 132 | 133 | echo "Listing files in dist dir..." 134 | ls -alRh "$PYAPP_DIST_DIR" 135 | -------------------------------------------------------------------------------- /tests/test_status_config.py: -------------------------------------------------------------------------------- 1 | """Providers main test module.""" 2 | 3 | from unittest.mock import patch 4 | 5 | from watchmaker.config.status import ( 6 | get_cloud_with_prereqs, 7 | get_non_cloud_providers, 8 | get_required_cloud_wo_prereqs, 9 | get_supported_cloud_w_prereqs, 10 | ) 11 | 12 | 13 | @patch( 14 | "watchmaker.config.status.get_cloud_with_prereqs", 15 | return_value=["aws", "azure"], 16 | ) 17 | def test_supported_cloud_w_prereqs(prereqs): 18 | """Test get required ids that are have prereqs.""" 19 | config_status = { 20 | "providers": [ 21 | { 22 | "key": "WatchmakerStatus", 23 | "required": False, 24 | "provider_type": "aws", 25 | }, 26 | { 27 | "key": "WatchmakerStatus", 28 | "required": False, 29 | "provider_type": "azure", 30 | }, 31 | { 32 | "key": "WatchmakerStatus", 33 | "required": False, 34 | "provider_type": "gcp", 35 | }, 36 | ], 37 | } 38 | 39 | ids = get_supported_cloud_w_prereqs(config_status) 40 | 41 | assert ids is not None 42 | assert "aws" in ids 43 | assert "azure" in ids 44 | assert "gcp" not in ids 45 | assert "none" not in ids 46 | 47 | 48 | @patch( 49 | "watchmaker.config.status.get_cloud_missing_prereqs", 50 | return_value=["aws", "azure"], 51 | ) 52 | def test_req_cloud_wo_prereqs(prereqs): 53 | """Test get required ids that are missing prereqs.""" 54 | config_status = { 55 | "providers": [ 56 | { 57 | "key": "WatchmakerStatus", 58 | "required": False, 59 | "provider_type": "aws", 60 | }, 61 | { 62 | "key": "WatchmakerStatus", 63 | "required": True, 64 | "provider_type": "azure", 65 | }, 66 | { 67 | "key": "WatchmakerStatus", 68 | "required": False, 69 | "provider_type": "gcp", 70 | }, 71 | ], 72 | } 73 | 74 | ids = get_required_cloud_wo_prereqs(config_status) 75 | 76 | assert ids is not None 77 | assert "aws" not in ids 78 | assert "azure" in ids 79 | assert "gcp" not in ids 80 | assert "none" not in ids 81 | 82 | 83 | @patch( 84 | "watchmaker.config.status.get_cloud_missing_prereqs", 85 | return_value=[], 86 | ) 87 | def test_no_req_cloud_wo_prereqs(prereqs): 88 | """Test get required ids that are missing prereqs.""" 89 | config_status = { 90 | "providers": [ 91 | { 92 | "key": "WatchmakerStatus", 93 | "required": True, 94 | "provider_type": "aws", 95 | }, 96 | { 97 | "key": "WatchmakerStatus", 98 | "required": True, 99 | "provider_type": "azure", 100 | }, 101 | { 102 | "key": "WatchmakerStatus", 103 | "required": False, 104 | "provider_type": "gcp", 105 | }, 106 | ], 107 | } 108 | 109 | assert not get_required_cloud_wo_prereqs(config_status) 110 | 111 | 112 | @patch( 113 | "watchmaker.config.status.SUPPORTED_CLOUD_PROVIDERS", 114 | [ 115 | {"provider": "aws", "has_prereq": True}, 116 | {"provider": "azure", "has_prereq": False}, 117 | ], 118 | ) 119 | def test_get_cloud_with_prereqs(): 120 | """Test get ids with prereqs.""" 121 | providers = get_cloud_with_prereqs() 122 | 123 | assert len(providers) == 1 124 | assert providers[0] == "aws" 125 | 126 | 127 | @patch( 128 | "watchmaker.config.status.SUPPORTED_CLOUD_PROVIDERS", 129 | [ 130 | {"provider": "aws", "has_prereq": True}, 131 | {"provider": "azure", "has_prereq": False}, 132 | ], 133 | ) 134 | def test_cloud_ids_missing_prereqs(): 135 | """Test get ids missing prereqs.""" 136 | providers = get_cloud_with_prereqs() 137 | 138 | assert len(providers) == 1 139 | assert providers[0] != "azure" 140 | 141 | 142 | @patch( 143 | "watchmaker.config.status.SUPPORTED_NON_CLOUD_PROVIDERS", 144 | [ 145 | {"provider": "file"}, 146 | {"provider": "db"}, 147 | ], 148 | ) 149 | def test_get_non_cloud_providers(): 150 | """Test get non cloud identifiers matching config.""" 151 | config_status = { 152 | "providers": [ 153 | { 154 | "key": "WatchmakerStatus", 155 | "required": False, 156 | "provider_type": "sqs", 157 | }, 158 | { 159 | "key": "WatchmakerStatus", 160 | "required": False, 161 | "provider_type": "file", 162 | }, 163 | ], 164 | } 165 | 166 | providers = get_non_cloud_providers(config_status) 167 | 168 | assert len(providers) == 1 169 | assert providers[0] == "file" 170 | 171 | 172 | @patch( 173 | "watchmaker.config.status.SUPPORTED_NON_CLOUD_PROVIDERS", 174 | [], 175 | ) 176 | def test_no_non_cloud_providers(): 177 | """Test empty SUPPORTED_NON_CLOUD_PROVIDERS.""" 178 | config_status = { 179 | "providers": [ 180 | { 181 | "key": "WatchmakerStatus", 182 | "required": False, 183 | "provider_type": "sqs", 184 | }, 185 | { 186 | "key": "WatchmakerStatus", 187 | "required": False, 188 | "provider_type": "file", 189 | }, 190 | ], 191 | } 192 | 193 | assert not get_non_cloud_providers(config_status) 194 | -------------------------------------------------------------------------------- /docs/troubleshooting/Linux/cloud-init.log.md: -------------------------------------------------------------------------------- 1 | ```{eval-rst} 2 | .. image:: /images/cropped-plus3it-logo-cmyk.png 3 | :width: 140px 4 | :alt: Powered by Plus3 IT Systems 5 | :align: right 6 | :target: https://www.plus3it.com 7 | ``` 8 |
9 | 10 | # The `/var/log/cloud-init.log` Log-File 11 | 12 | This is the default location where the Red Hat packaged version of the `cloud-init` service for Enterprise Linux 6 and 7 writes _all_ of its log-output to – on RHEL 8+, logging data is split-out across this file and the `/var/log/cloud-init-output.log` file. All automation directly-initiated through `cloud-init` and that emits STDOUT and/or STDERR messages will be duplicated here. 13 | 14 | Primary diagnostic-use with respect to execution of `watchmaker` will be in tracking errors emitted during _preparation to execute_ `watchmaker`. If the `watchmaker` process fails to start (meaning that `/var/log/watchmaker/watchmaker.log` is never created), this would be a good location to find _why_ `watchmaker` failed to start. 15 | 16 | Useful string-searches for locating executional points-of-interest ("landmarks") will be (ordered most- to least-useful): 17 | 18 | * `: FAIL: ` 19 | * `/var/lib/cloud/instance/script` 20 | * `/var/lib/cloud/instance` 21 | * `: SUCCESS: ` 22 | 23 | By far, the search for `: FAIL:` will be the most important in uncovering errors. The other searches will mostly be of use in progress-tracking and verifying expected event-sequencing[^1]. 24 | 25 | ## Example Failure 26 | 27 | Typically, searching for "`: FAIL:` will bring the file-cursor to a logged-block similar to: 28 | 29 | ~~~bash 30 | 2023-06-21 11:12:36,078 - subp.py[DEBUG]: Unexpected error while running command. 31 | Command: ['/var/lib/cloud/instance/scripts/00_script.sh'] 32 | Exit code: 1 33 | Reason: - 34 | Stdout: - 35 | Stderr: - 36 | 2023-06-21 11:12:36,078 - cc_scripts_user.py[WARNING]: Failed to run module scripts-user (scripts in /var/lib/cloud/instance/scripts) 37 | 2023-06-21 11:12:36,078 - handlers.py[DEBUG]: finish: modules-final/config-scripts-user: FAIL: running config-scripts-user with frequency once-per-instance 38 | 2023-06-21 11:12:36,078 - util.py[WARNING]: Running module scripts-user () failed 39 | 2023-06-21 11:12:36,079 - util.py[DEBUG]: Running module scripts-user () failed 40 | Traceback (most recent call last): 41 | File "/usr/lib/python3.6/site-packages/cloudinit/stages.py", line 1090, in _run_modules 42 | run_name, mod.handle, func_args, freq=freq 43 | File "/usr/lib/python3.6/site-packages/cloudinit/cloud.py", line 55, in run 44 | return self._runners.run(name, functor, args, freq, clear_on_fail) 45 | File "/usr/lib/python3.6/site-packages/cloudinit/helpers.py", line 185, in run 46 | results = functor(*args) 47 | File "/usr/lib/python3.6/site-packages/cloudinit/config/cc_scripts_user.py", line 44, in handle 48 | subp.runparts(runparts_path) 49 | File "/usr/lib/python3.6/site-packages/cloudinit/subp.py", line 426, in runparts 50 | % (len(failed), ",".join(failed), len(attempted)) 51 | RuntimeError: Runparts: 1 failures (00_script.sh) in 1 attempted commands 52 | ~~~ 53 | 54 | In this case, the failure happened during the execution of the userdata-script, `/var/lib/cloud/instance/scripts/00_script.sh`. Even if the script hasn't logged anything directly useful in this log file or hasn't even been configured to log its own activities any where, knowing that it was during the execution of this file is useful. 55 | 56 | 1. The provisioning-administrator knows where in the `cloud-init` automation-sequence things failed 57 | 2. One can look in other logs for actionable diagnostic-information 58 | 3. If there's no such information in other log files, one can hand-execute the failing script to see if the error can be reproduced (and in a way that assists the provisioning-administator with isolating the source of the failure) 59 | 60 | For the third point, if the failure is in a BASH script, executing the script with the diagnostic flag set (e.g., `bash -x /var/lib/cloud/instance/scripts/00_script.sh`) one may be able to see where the script fails. 61 | 62 | Similarly, if hand-execution of the script _succeeds_ it can point to the script making incorrect assumptions about the `cloud-init` managed execution environment. This can include things like: 63 | 64 | - Lack of necessary environment variables 65 | - Improperly defined environment-variables 66 | - Attempts to execute commands that require a controlling-TTY (i.e., an interactive-login shell) 67 | - Attempting to do something that the instance's security posture blocks[^2]. 68 | 69 | Note that comparing execution via `cloud-init` versus execution from an interactive-shell works whether the script is written in BASH or some other interpreted language. 70 | 71 | [^1]: Event-sequencing issues most-frequently happen when a userData payload delivers two or more scripts. When multiple scripts are specified in a userData payload, they are not necessarily executed in the same order they're specified in the userData text-stream. Instead, `cloud-init` executes scripts placed into the `/var/lib/cloud/instance/scripts/` directory in alphabetical order. Thus, if one _needs_ the scripts to execute in a specific order, it is important to carefully name them such that that happens (e.g., `00_script` and `01_script` would result in the `00_`-prefixed script executing prior the `01_`-prefixed script) 72 | [^2]: SELinux can be especially problematic for processes started by `cloud-init`. For example, the `firewall-cmd` utility is not directly usable. `cloud-init` scripts would need to either issue a `setenforce 0` before invoking the command or use the alternate `firewall-offline-command` 73 | -------------------------------------------------------------------------------- /docs/files/bootstrap/watchmaker-bootstrap.ps1: -------------------------------------------------------------------------------- 1 | [CmdLetBinding()] 2 | Param( 3 | [String]$PythonUrl = "https://www.python.org/ftp/python/3.10.11/python-3.10.11-amd64.exe" 4 | , 5 | [String]$GitUrl 6 | , 7 | [String]$RootCertUrl 8 | ) 9 | $__ScriptName = "watchmaker-boostrap.ps1" 10 | 11 | # Location to save files. 12 | $SaveDir = ${Env:Temp} 13 | 14 | function Install-Msi { 15 | Param( [String]$Installer, [String[]]$InstallerArgs ) 16 | $Arguments = @() 17 | $Arguments += "/i" 18 | $Arguments += "`"${Installer}`"" 19 | $Arguments += $InstallerArgs 20 | Write-Verbose "Installing $Installer" 21 | $ret = Start-Process "msiexec.exe" -ArgumentList ${Arguments} -NoNewWindow -PassThru -Wait 22 | } 23 | 24 | function Install-Exe { 25 | Param( [String]$Installer, [String[]]$InstallerArgs ) 26 | Write-Verbose "Installing $Installer" 27 | $ret = Start-Process "${Installer}" -ArgumentList ${InstallerArgs} -NoNewWindow -PassThru -Wait 28 | } 29 | 30 | function Download-File { 31 | Param( [string]$Url, [string]$SavePath ) 32 | # Download a file, if it doesn't already exist. 33 | if( !(Test-Path ${SavePath} -PathType Leaf) ) { 34 | [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::SystemDefault 35 | $SecurityProtocolTypes = @([Net.SecurityProtocolType].GetEnumNames()) 36 | if ("Tls11" -in $SecurityProtocolTypes) { 37 | [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls11 38 | } 39 | if ("Tls12" -in $SecurityProtocolTypes) { 40 | [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 41 | } 42 | 43 | (New-Object System.Net.WebClient).DownloadFile(${Url}, ${SavePath}) 44 | Write-Verbose "Downloaded ${Url} to ${SavePath}" 45 | } 46 | } 47 | 48 | function Reset-EnvironmentVariables { 49 | foreach( $Level in "Machine", "User" ) { 50 | [Environment]::GetEnvironmentVariables(${Level}).GetEnumerator() | % { 51 | # For Path variables, append the new values, if they're not already in there. 52 | if($_.Name -match 'Path$') { 53 | $_.Value = ($((Get-Content "Env:$($_.Name)") + ";$($_.Value)") -split ';' | Select -unique) -join ';' 54 | } 55 | $_ 56 | } | Set-Content -Path { "Env:$($_.Name)" } 57 | } 58 | } 59 | 60 | function Install-Python { 61 | $PythonFile = "${SaveDir}\$(${PythonUrl}.split("/")[-1])" 62 | 63 | Download-File -Url ${PythonUrl} -SavePath ${PythonFile} 64 | 65 | if ($PythonFile -match "^.*msi$") { 66 | $Arguments = @() 67 | $Arguments += "/qn" 68 | $Arguments += "ALLUSERS=1" 69 | $Arguments += "ADDLOCAL=ALL" 70 | Install-Msi -Installer ${PythonFile} -InstallerArgs ${Arguments} 71 | } 72 | elseif ($PythonFile -match "^.*exe$") { 73 | $Arguments = @() 74 | $Arguments += "/quiet" 75 | $Arguments += "InstallAllUsers=1" 76 | $Arguments += "PrependPath=1" 77 | Install-Exe -Installer ${PythonFile} -InstallerArgs ${Arguments} 78 | } 79 | 80 | Write-Verbose "Installed Python" 81 | } 82 | 83 | function Install-Git { 84 | $GitFile = "${SaveDir}\$(${GitUrl}.split("/")[-1])" 85 | 86 | Download-File -Url ${GitUrl} -SavePath ${GitFile} 87 | 88 | $Arguments = @() 89 | $Arguments += "/VERYSILENT" 90 | $Arguments += "/NOCANCEL" 91 | $Arguments += "/NORESTART" 92 | $Arguments += "/SAVEINF=${SaveDir}\git_params.txt" 93 | Install-Exe -Installer ${GitFile} -InstallerArgs ${Arguments} 94 | 95 | Write-Verbose "Installed Git" 96 | } 97 | 98 | function Import-509Certificate { 99 | Param( [String]$CertFile, [String]$CertRootStore, [String]$CertStore ) 100 | Write-Verbose "Importing certificate: ${CertFile} ..." 101 | $Pfx = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2 102 | $Pfx.import($CertFile) 103 | $Store = New-Object System.Security.Cryptography.X509Certificates.x509Store(${CertStore},${CertRootStore}) 104 | $Store.open("MaxAllowed") 105 | $Store.add($Pfx) 106 | $Store.close() 107 | } 108 | 109 | function Install-RootCerts { 110 | Param( [string]$RootCertHost ) 111 | $CertDir = "${SaveDir}\certs-$(${RootCertHost}.Replace(`"http://`",`"`"))" 112 | Write-Verbose "Creating directory for certificates at ${CertDir}." 113 | New-Item -Path ${CertDir} -ItemType "directory" -Force -WarningAction SilentlyContinue | Out-Null 114 | 115 | Write-Verbose "... Checking for certificates hosted by: ${RootCertHost} ..." 116 | $CertUrls = @((Invoke-WebRequest -Uri ${RootCertHost}).Links | Where { $_.href -Match ".*\.cer$" } | ForEach-Object { ${RootCertHost} + $_.href }) 117 | 118 | Write-Verbose "... Found $(${CertUrls}.count) certificate(s) ..." 119 | Write-Verbose "... Downloading and importing certificate(s) ..." 120 | foreach( $UrlItem in ${CertUrls} ) { 121 | $CertFile = "${CertDir}\$((${UrlItem}.split('/'))[-1])" 122 | Download-File ${UrlItem} ${CertFile} 123 | if( ${CertFile} -match ".*root.*" ) { 124 | Import-509Certificate ${CertFile} "LocalMachine" "Root" 125 | Write-Verbose "Imported trusted root CA certificate: ${CertFile}" 126 | } else { 127 | Import-509Certificate ${CertFile} "LocalMachine" "CA" 128 | Write-Verbose "Imported intermediate CA certificate: ${CertFile}" 129 | } 130 | } 131 | Write-Verbose "... Completed import of certificate(s) from: ${RootCertHost}" 132 | } 133 | 134 | # Main 135 | 136 | if( ${RootCertUrl} ) { 137 | # Download and install the root certificates. 138 | Write-Verbose "Root certificates host url is ${RootCertUrl}" 139 | Install-RootCerts ${RootCertUrl} 140 | } 141 | 142 | # Install Python 143 | Write-Verbose "Python will be installed from ${PythonUrl}" 144 | Install-Python 145 | 146 | if( ${GitUrl} ) { 147 | # Download and install git 148 | Write-Verbose "Git will be installed from ${GitUrl}" 149 | Install-Git 150 | } 151 | 152 | Reset-EnvironmentVariables 153 | Write-Verbose "Reset the PATH environment for this shell" 154 | 155 | if ("$Env:TEMP".TrimEnd("\") -eq "${Env:windir}\System32\config\systemprofile\AppData\Local\Temp") { 156 | $Env:TEMP, $Env:TMP = "${Env:windir}\Temp", "${Env:windir}\Temp" 157 | Write-Verbose "Forced TEMP envs to ${Env:windir}\Temp" 158 | } 159 | 160 | Write-Verbose "${__ScriptName} complete!" 161 | -------------------------------------------------------------------------------- /src/watchmaker/config/__init__.py: -------------------------------------------------------------------------------- 1 | """Config module.""" 2 | 3 | import collections 4 | import logging 5 | import re 6 | 7 | import yaml 8 | from compatibleversion import check_version 9 | 10 | try: 11 | from importlib.resources import files 12 | except ImportError: 13 | from importlib_resources import files 14 | 15 | import watchmaker.utils.imds.detect 16 | from watchmaker.config.status import is_valid 17 | from watchmaker.exceptions import ( 18 | InvalidRegexPatternError, 19 | StatusConfigError, 20 | WatchmakerError, 21 | ) 22 | from watchmaker.utils import urllib_utils 23 | 24 | log = logging.getLogger(__name__) 25 | 26 | 27 | def get_configs(system, worker_args, config_path=None): # noqa: C901 28 | """ 29 | Read and validate configuration data for installation. 30 | 31 | Returns: 32 | :obj:`collections.OrderedDict`: Returns the data from the the YAML 33 | configuration file, scoped to the value of ``system`` and 34 | merged with the value of the ``"All"`` key. 35 | 36 | """ 37 | data = "" 38 | if not config_path: 39 | log.warning("User did not supply a config. Using the default config.") 40 | data = files("watchmaker.static").joinpath("config.yaml").read_text() 41 | else: 42 | log.info("User supplied config being used.") 43 | 44 | # Convert a local config path to a URI 45 | config_path = watchmaker.utils.uri_from_filepath(config_path) 46 | 47 | # Get the raw config data 48 | try: 49 | data = watchmaker.utils.urlopen_retry(config_path).read() 50 | except (ValueError, urllib_utils.error.URLError): 51 | msg = ( 52 | f'Could not read config file from the provided value "{config_path}"! ' 53 | "Check that the config is available." 54 | ) 55 | log.critical(msg) 56 | raise 57 | 58 | config_full = yaml.safe_load(data) 59 | try: 60 | config_all = config_full.get("all", []) 61 | config_system = config_full.get(system, []) 62 | config_status = config_full.get("status", None) 63 | config_version_specifier = config_full.get("watchmaker_version", None) 64 | except AttributeError: 65 | msg = "Malformed config file. Must be a dictionary." 66 | log.critical(msg) 67 | raise 68 | 69 | # If both config and config_system are empty, raise 70 | if not config_system and not config_all: 71 | msg = "Malformed config file. No workers for this system." 72 | log.critical(msg) 73 | raise WatchmakerError(msg) 74 | 75 | if config_version_specifier and not check_version( 76 | watchmaker.__version__, 77 | config_version_specifier, 78 | ): 79 | msg = ( 80 | f"Watchmaker version {watchmaker.__version__} is not compatible " 81 | f"with the config file (watchmaker_version = {config_version_specifier})" 82 | ) 83 | log.critical(msg) 84 | raise WatchmakerError(msg) 85 | 86 | # Merge the config data, preserving the listed order of workers. 87 | # The worker order from config_system has precedence over config_all. 88 | # This is managed by adding config_system to the config first, using 89 | # the loop order, e.g. config_system + config_all. In the loop, if the 90 | # worker is already in the config, it is always the worker from 91 | # config_system. 92 | # To also preserve precedence of worker options from config_system, the 93 | # worker_config from config_all is updated with the config from 94 | # config_system, then the config is replaced with the worker_config. 95 | config = collections.OrderedDict() 96 | for worker in config_system + config_all: 97 | try: 98 | # worker is a single-key dict, where the key is the name of the 99 | # worker and the value is the worker parameters. we need to 100 | # test if the worker is already in the config, but a dict is 101 | # is not hashable so cannot be tested directly with 102 | # `if worker not in config`. this bit of ugliness extracts the 103 | # key and its value so we can use them directly. 104 | worker_name, worker_config = next(iter(worker.items())) 105 | if worker_name not in config: 106 | # Add worker to config 107 | config[worker_name] = {"config": worker_config} 108 | log.debug("%s config: %s", worker_name, worker_config) 109 | else: 110 | # Worker is present in both config_system and config_all, 111 | # config[worker_name]['config'] is from config_system, 112 | # worker_config is from config_all 113 | worker_config.update(config[worker_name]["config"]) 114 | config[worker_name]["config"] = worker_config 115 | log.debug("%s extra config: %s", worker_name, worker_config) 116 | # Need to (re)merge cli worker args so they override 117 | config[worker_name]["__merged"] = False 118 | if not config[worker_name].get("__merged"): 119 | # Merge worker_args into config params 120 | config[worker_name]["config"].update(worker_args) 121 | config[worker_name]["__merged"] = True 122 | except Exception: # noqa: PERF203 123 | msg = f"Failed to merge worker config; worker={worker}" 124 | log.critical(msg) 125 | raise 126 | 127 | log.debug("Command-line arguments merged into worker configs: %s", worker_args) 128 | 129 | validate_computer_name_pattern(config) 130 | 131 | if not is_valid(config_status): 132 | log.error("Status config is invalid %s", config_status) 133 | raise StatusConfigError(config_status) 134 | 135 | return config, config_status 136 | 137 | 138 | def validate_computer_name_pattern(config): 139 | """Ensure pattern is valid and will match entire computer name.""" 140 | computer_name_pattern = ( 141 | config.get("salt", {}).get("config", {}).get("computer_name_pattern", {}) 142 | ) 143 | 144 | if computer_name_pattern: 145 | try: 146 | re.compile(computer_name_pattern) 147 | except re.error as exc: 148 | raise InvalidRegexPatternError(computer_name_pattern) from exc 149 | --------------------------------------------------------------------------------