├── tests ├── __init__.py └── noxfile.py ├── src └── vaulti_ansible │ ├── __init__.py │ ├── __about__.py │ ├── __main__.py │ └── vaulti.py ├── example ├── .foo_vault_pass.txt ├── .default_vault_pass.txt └── example_data.yml ├── .gitignore ├── docs ├── multiline_variables.md ├── vault_ids.md ├── git_diff_tricks.md └── installation_and_basic_usage.md ├── LICENSE ├── pyproject.toml ├── .github └── workflows │ └── autopublish_pypi.yml └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/vaulti_ansible/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/.foo_vault_pass.txt: -------------------------------------------------------------------------------- 1 | foo 2 | -------------------------------------------------------------------------------- /example/.default_vault_pass.txt: -------------------------------------------------------------------------------- 1 | default 2 | -------------------------------------------------------------------------------- /src/vaulti_ansible/__about__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.11" 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | .venv/ 3 | tests/__pycache__/ 4 | src/vaulti_ansible/__pycache__/ 5 | -------------------------------------------------------------------------------- /src/vaulti_ansible/__main__.py: -------------------------------------------------------------------------------- 1 | from .vaulti import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /docs/multiline_variables.md: -------------------------------------------------------------------------------- 1 | # Multiline variables 2 | 3 | vaulti will print the decrypted content in one of several ways: 4 | 5 | - If the encrypted content does not contain a newline (`\n`), it will print it on the same line, like 6 | 7 | ```yaml 8 | key: !ENCRYPT value 9 | ``` 10 | 11 | - If the encrypted content contains a newline (`\n`), it will use literal block scalars (`|`) and break it over multiple lines for readability 12 | 13 | ```yaml 14 | key: !ENCRYPT | 15 | this is some 16 | multiline content 17 | that is encrypted 18 | ``` 19 | 20 | - If the encrypted content *does not* contain a newline at the end, it will additionally add a block chomping indicator (hyphen): 21 | 22 | ```yaml 23 | key: !ENCRYPT |- 24 | ------BEGIN CERT------ 25 | abcdefghijklmnopqrstuvqxyz 26 | nopqrstuvwxyzabcdefghidklm 27 | abcdefghijklmnopqrstuvqxyz 28 | ------END CERT------ 29 | ``` 30 | 31 | You can also use this while creating or editing secrets, and it will encrypt it correctly for you. 32 | -------------------------------------------------------------------------------- /docs/vault_ids.md: -------------------------------------------------------------------------------- 1 | # Vault ids 2 | 3 | ## Usage 4 | 5 | ```shell 6 | # To see the variables encrypted with the foo vault ID, load it too, either being prompted for the password, or referring to a file 7 | vaulti example/example_data.yml --vault-id foo@prompt 8 | vaulti example/example_data.yml --vault-id foo@example/.foo_vault_pass.txt 9 | ``` 10 | 11 | ## Explanations 12 | 13 | Secrets decrypted with the non-default ID will be shown in the tag as `!ENCRYPT:mylabel`. You can also set these labels yourself in the edit 14 | mode, as long as you actually loaded the relevant vault-id when starting the utility. 15 | 16 | **WARNING**: The labels are there to help *you* when prompted, but ansible-vault will try all of the keys when decrypting. 17 | So if you have two vault-ids, but you swap the passwords on the prompt, it will still decrypt just fine. However, when you save and quit, 18 | now you'll encrypt the variables with the swapped passwords instead, which will definitely lead to confusion. 19 | To avoid this, make sure to set the environment variable `ANSIBLE_VAULT_ID_MATCH=true`, or `vault_id_match = True` in ansible.cfg. 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Ove 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "vaulti_ansible" 7 | dynamic = [ 8 | "version", 9 | ] 10 | authors = [ 11 | { name="Ove", email="oveee92@gmail.com" }, 12 | ] 13 | ## Don't list ansible as a dependency, because pip's upgrade strategy is weird; In ansible we might 14 | ## want the execution environments to be on a certain version of ansible-core, and mentioning them 15 | ## here in any way will upgrade it to the latest. Just add it to the README for completeness. 16 | dependencies = [ 17 | "ruamel.yaml>=0.16.6", # Won't work before this version due to TaggedScalar changes 18 | # "ansible>=2.4.0.0", # Won't work for older versions due to pycrypto 19 | ] 20 | keywords = ["ansible", "ansible-vault"] 21 | description = "Utility for Ansible Vault inline-encrypted variables" 22 | readme = "README.md" 23 | requires-python = ">=3.8" # __future__ is not not available before python 3.7, and ansible seems to skip 3.7, so we just use 3.8 24 | classifiers = [ 25 | "Development Status :: 4 - Beta", 26 | "Programming Language :: Python :: 3", 27 | "License :: OSI Approved :: MIT License", 28 | "Operating System :: POSIX :: Linux", 29 | "Framework :: Ansible", 30 | "Intended Audience :: System Administrators", 31 | "Topic :: Security :: Cryptography", 32 | "Topic :: System :: Systems Administration", 33 | "Topic :: Utilities", 34 | ] 35 | license = {text = "MIT"} 36 | 37 | [tool.hatch.version] 38 | path = "src/vaulti_ansible/__about__.py" 39 | 40 | [project.scripts] 41 | vaulti = "vaulti_ansible.vaulti:main" 42 | 43 | [project.urls] 44 | Homepage = "https://github.com/oveee92/vaulti" 45 | Repository = "https://github.com/oveee92/vaulti" 46 | Issues = "https://github.com/oveee92/vaulti/issues" 47 | -------------------------------------------------------------------------------- /docs/git_diff_tricks.md: -------------------------------------------------------------------------------- 1 | # Git diff tricks 2 | 3 | You can set up a script that decrypts the contents of your files so that your git diff is actually useful for ansible-vault, 4 | not just a random encrypted-armored vault string. 5 | 6 | Using a custom conversion of the data is called `textconv` in git. 7 | 8 | It has several parts, and is set up so that it only works for you, and doesn't mess up the config of your team. 9 | 10 | ## Custom wrapper script 11 | 12 | The script basically tries to verify whether it is a yaml file and contains the string `!vault`. If it does, it 13 | passes it to vaulti with the `--view|-r` option, and if not, it just cats it out as it is. 14 | 15 | Content of `~/.local/bin/vaulti_wrapper.sh`: 16 | 17 | ```shell 18 | #!/bin/bash 19 | 20 | # Check if the file ends with .yml or .yaml 21 | if [[ "$1" == *.yml || "$1" == *.yaml ]]; then 22 | # If the file contains the magic string "!vault", run the custom command 23 | if grep -q "!vault" "$1"; then 24 | vaulti -r "$1" 25 | else 26 | # If not, just cat the file 27 | cat "$1" 28 | fi 29 | else 30 | # If the file is not a .yml or .yaml file, output it unchanged 31 | cat "$1" 32 | fi 33 | ``` 34 | 35 | ## Global gitconfig 36 | 37 | Then you have to specify a custom textconv on the "vaulti" diff, in your home directory. 38 | 39 | Content of ~/.gitconfig 40 | ```ini 41 | [diff "vaulti] 42 | textconv = ~/.local/bin/vaulti_wrapper.sh 43 | ``` 44 | 45 | ## Git attributes 46 | 47 | You have to specify which files should use the custom textconv. I just select all files. 48 | Put it in `.git/info/attributes` so that your change stays local and won't affect your team. 49 | 50 | Content of `/.git/info/attributes` 51 | 52 | ```ini 53 | * diff=vaulti 54 | ``` 55 | 56 | ## General 57 | 58 | Now when you use `git diff`, it will (of course) be a little slower than normal, but will now 59 | show you the diff of the decrypted content, which might be very useful. 60 | 61 | If you want to see the actual content again, just remove the content of `/.git/info/attributes`. 62 | 63 | If you want to see the actual content again temporarily, just use `git diff --no-textconv` 64 | 65 | ## Future 66 | 67 | This is not a very "stable" configuration, and might have a lot of caveats. It might break the git diff functionality if you 68 | haven't set up the `ANSIBLE_VAULT_PASSWORD_FILE` variable, it might not work for vault-encrypted (whole) files, 69 | but is very useful in cases where you are changing lots of encrypted variables and want sanity checks every now and then. 70 | 71 | Use it if you find it helpful! 72 | -------------------------------------------------------------------------------- /example/example_data.yml: -------------------------------------------------------------------------------- 1 | --- 2 | enc_variable: !vault | 3 | $ANSIBLE_VAULT;1.2;AES256;foo 4 | 33396464623135356139613062323664623037316134343663346365643036666431316665663731 5 | 6465666631363965353532663962633832303663653230640a663939643063643565306634343933 6 | 38623861383338366436626234383662316561316439666237363034616138383833323465376638 7 | 6538383137376133300a623965623338373134346361396539343462396137346430656162373961 8 | 3335 9 | 10 | 11 | enc_foo: !vault | 12 | $ANSIBLE_VAULT;1.2;AES256;foo 13 | 30613265366233323838316535623334333634323038383832366436646235313465303966353966 14 | 3238366432623864613461353932323762373265313565360a623732626664663039616338326331 15 | 34636237653634353164636333623361636463626139613064356566616465323062356338373937 16 | 3135646662393063630a373230313264353937393865613866303766643034383766386230336431 17 | 3737 18 | 19 | 20 | plain_variable: gonna 21 | 22 | list_variable: # Encrypted variables in a list probably isn't really valid for ansible though 23 | - !vault | 24 | $ANSIBLE_VAULT;1.1;AES256 25 | 62663730323536326530356331356166653763333730303933633537396431386334643361646364 26 | 3666626234653430366433656239313334366434316531300a383865323166356338303165633834 27 | 31643165316230303538656134626666646230616637656633393038356165376139356533623030 28 | 6562346266303462660a613935363238326663363662363037333234666365663661646339373065 29 | 3233 30 | list_of_dicts: 31 | - plaintext: you 32 | encrypted: !vault | 33 | $ANSIBLE_VAULT;1.1;AES256 34 | 61653430333031326465393735633862626164353064396533356264333437336133626566306464 35 | 6230383963646631616434383639636436313135326133640a353532633735303037353032653937 36 | 34333231383734663761653336383336373433323662396532386466626332656666626366666665 37 | 3966636139363133340a303838313738306338623361663862333864383234343465636564343539 38 | 3763 39 | 40 | 41 | somedict: 42 | boo_bla: !vault | 43 | $ANSIBLE_VAULT;1.1;AES256 44 | 38363638643939393861663438636335376634333033616530326231373737363538303161663530 45 | 3934326138356561343831623330626230346537343336630a363363366535386462393934613438 46 | 62356139306537616337613764613961336262316461393939393138343663333666613263633333 47 | 3534343933646535370a363561666339346264363738663463613033393231393630656164616130 48 | 6661 49 | 50 | test: !vault | 51 | $ANSIBLE_VAULT;1.1;AES256 52 | 38623034636234373831316234613063326266666536336232366333333961626262643564623462 53 | 3233396366626366353266303139323239653361623536360a656131656233306635346230313130 54 | 65666464383238303861326131623837643533343436643534363933376466383735626531333337 55 | 6130353761653561390a336632336230336336363332313937313039326435656330313035313463 56 | 3630 57 | 58 | deep_nest: 59 | - my_list: 60 | - foo: you 61 | bar: !vault | 62 | $ANSIBLE_VAULT;1.1;AES256 63 | 65663264326536386266333631316437616462366536383738383931336637666435613666316437 64 | 3363353965626339653334623433626138333838646631330a366165643264636538643433663137 65 | 31356133643562343162653162393564373536363231613566633839383436316166666438626133 66 | 6566316135303032300a633831353934396134353263333636613539326639396539373963343334 67 | 6463 68 | 69 | ... 70 | -------------------------------------------------------------------------------- /.github/workflows/autopublish_pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI 2 | 3 | on: push 4 | 5 | jobs: 6 | build: 7 | name: Build distribution 📦 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v4 12 | with: 13 | persist-credentials: false 14 | - name: Set up Python 15 | uses: actions/setup-python@v5 16 | with: 17 | python-version: "3.x" 18 | - name: Install pypa/build 19 | run: >- 20 | python3 -m 21 | pip install 22 | build 23 | --user 24 | - name: Build a binary wheel and a source tarball 25 | run: python3 -m build 26 | - name: Store the distribution packages 27 | uses: actions/upload-artifact@v4 28 | with: 29 | name: python-package-distributions 30 | path: dist/ 31 | 32 | publish-to-pypi: 33 | name: >- 34 | Publish Python 🐍 distribution 📦 to PyPI 35 | if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes 36 | needs: 37 | - build 38 | runs-on: ubuntu-latest 39 | environment: 40 | name: pypi 41 | url: https://pypi.org/p/vaulti-ansible # Replace with your PyPI project name 42 | permissions: 43 | id-token: write # IMPORTANT: mandatory for trusted publishing 44 | 45 | steps: 46 | - name: Download all the dists 47 | uses: actions/download-artifact@v4 48 | with: 49 | name: python-package-distributions 50 | path: dist/ 51 | - name: Publish distribution 📦 to PyPI 52 | uses: pypa/gh-action-pypi-publish@release/v1 53 | 54 | github-release: 55 | name: >- 56 | Sign the Python 🐍 distribution 📦 with Sigstore 57 | and upload them to GitHub Release 58 | needs: 59 | - publish-to-pypi 60 | runs-on: ubuntu-latest 61 | 62 | permissions: 63 | contents: write # IMPORTANT: mandatory for making GitHub Releases 64 | id-token: write # IMPORTANT: mandatory for sigstore 65 | 66 | steps: 67 | - name: Download all the dists 68 | uses: actions/download-artifact@v4 69 | with: 70 | name: python-package-distributions 71 | path: dist/ 72 | - name: Sign the dists with Sigstore 73 | uses: sigstore/gh-action-sigstore-python@v3.0.0 74 | with: 75 | inputs: >- 76 | ./dist/*.tar.gz 77 | ./dist/*.whl 78 | - name: Create GitHub Release 79 | env: 80 | GITHUB_TOKEN: ${{ github.token }} 81 | run: >- 82 | gh release create 83 | "$GITHUB_REF_NAME" 84 | --repo "$GITHUB_REPOSITORY" 85 | --notes "" 86 | - name: Upload artifact signatures to GitHub Release 87 | env: 88 | GITHUB_TOKEN: ${{ github.token }} 89 | # Upload to GitHub Release using the `gh` CLI. 90 | # `dist/` contains the built packages, and the 91 | # sigstore-produced signatures and certificates. 92 | run: >- 93 | gh release upload 94 | "$GITHUB_REF_NAME" dist/** 95 | --repo "$GITHUB_REPOSITORY" 96 | 97 | #publish-to-testpypi: 98 | # name: Publish Python 🐍 distribution 📦 to TestPyPI 99 | # needs: 100 | # - build 101 | # runs-on: ubuntu-latest 102 | 103 | # environment: 104 | # name: testpypi 105 | # url: https://test.pypi.org/p/vaulti-ansible 106 | 107 | # permissions: 108 | # id-token: write # IMPORTANT: mandatory for trusted publishing 109 | 110 | # steps: 111 | # - name: Download all the dists 112 | # uses: actions/download-artifact@v4 113 | # with: 114 | # name: python-package-distributions 115 | # path: dist/ 116 | # - name: Publish distribution 📦 to TestPyPI 117 | # uses: pypa/gh-action-pypi-publish@release/v1 118 | # with: 119 | # repository-url: https://test.pypi.org/legacy/ 120 | -------------------------------------------------------------------------------- /docs/installation_and_basic_usage.md: -------------------------------------------------------------------------------- 1 | # Installation and basic usage 2 | 3 | ## Installation 4 | 5 | The dists have been published to PyPi, you can easily install it with `pip install vaulti-ansible`. 6 | 7 | Alternatively, you can clone this repo and use another installation method: 8 | 9 | ```shell 10 | # You can test this by cloning the repo, cd into it 11 | git clone https://github.com/oveee92/vaulti.git && cd vaulti 12 | 13 | # then EITHER install it with pip to get it installed into your PATH and as a python module 14 | pip install . 15 | vaulti example/example_data.yml 16 | # OR put it somewhere in the PATH yourself 17 | cp .src/vaulti_ansible/vaulti.py ~/.local/bin/vaulti 18 | vaulti example/example_data.yml 19 | # OR just use it directly without "installing" it 20 | ./src/vaulti_ansible/vaulti.py example/example_data.yml 21 | 22 | ``` 23 | 24 | ## Simple example 25 | 26 | Now you can set up and use it. Example here works in the git directory if you used `git clone` above, 27 | but you can very easily just create your own files to test with. 28 | 29 | ```shell 30 | # If you want to use a password file, you can set it as a variable 31 | export ANSIBLE_VAULT_PASSWORD_FILE=example/.default_vault_pass.txt 32 | # OR specify it on the command line 33 | vaulti example/example_data.yml --vault-password-file example/.default_vault_pass.txt 34 | ``` 35 | 36 | Make some changes to existing variables, create some new ones or remove some tags. 37 | 38 | Save and quit, then open it regularly to see what changed, or just run git diff to see what happened. 39 | 40 | ```shell 41 | git diff example_encrypted_data.yaml 42 | ``` 43 | 44 | ## General usage 45 | 46 | You can use the standard ansible methods of defining a vault password or vault password file, like `--ask-vault-pass` parameter, 47 | `ANSIBLE_VAULT_PASSWORD_FILE` environment variable and `--vault-id`. 48 | 49 | There are some quality of life features built in, such as: 50 | 51 | - if you edit the file to some invalid yaml, you'll get the chance to re-open the file and try again 52 | - ditto if you try to encrypt with a vault id that you didn't load when starting 53 | - if you comment out a line while it is decrypted, it will not be reencrypted, but it will produce a warning. 54 | 55 | Variable files that could not be decrypted for whatever reason, get a tag indicating the problem, but is left untouched after exiting. 56 | 57 | ## Available tags 58 | 59 | The list of tags, both for success and failure, are currently: 60 | 61 | ### Success 62 | 63 | - `!ENCRYPT` : Variables that have been decrypted, and will be reencrypted when you close the editor 64 | - `!ENCRYPT:[label]`, for example `!ENCRYPT:foo`: Indicates that this value was decrypted with a specific vault-id label. 65 | 66 | ### Failure 67 | 68 | **Warning**: Do not change these during the editing process. Changing the value of encrypted strings will almost certainly corrupt them, 69 | and changing the tag value to something else might subject them to further processing (for example encrypting the encrypted content). 70 | 71 | - `!VAULT_FORMAT_ERROR` : Variables that could not be parsed due to ansible-vault rejecting the format. Ensure it was copypasted correctly, with no trailing whitespace characters. It will revert to the original `!vault` tag/value untouched after you close the editor. 72 | - `!UNKNOWN_VAULT_ID_LABEL` : Variables that could not be decrypted, most likely because you did not load/specify the relevant vault id. It will revert to the original `!vault` tag/value untouched after you close the editor. 73 | - `!COULD_NOT_DECRYPT` : Variables that could not be decrypted for any other reason, but most probably because you specified the wrong password. It will revert to the original `!vault` tag/value untouched after you close the editor. 74 | - `![any tag]:[label]`, for example `!UNKNOWN_VAULT_ID_LABEL:foo`: The label just indicates the vault-id fetched from the payload string. If nothing is specified, it uses the default vault id. 75 | -------------------------------------------------------------------------------- /tests/noxfile.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import nox 4 | import shutil 5 | import filecmp 6 | from ruamel.yaml import YAML 7 | from ruamel.yaml.comments import TaggedScalar 8 | from ansible.parsing.vault import (VaultLib, VaultSecret) 9 | 10 | @nox.session(name="Edit keys without changing encrypted content?") 11 | def test_unencrypted_keyname_replace(session): 12 | """ This tests for whether keys can be updated, and whether the vaulted value 13 | remains the same when not being changed """ 14 | 15 | vault_pass = "default" 16 | yaml_content_initial = """--- 17 | Never: gonna 18 | give: you 19 | up: !vault | 20 | $ANSIBLE_VAULT;1.1;AES256 21 | 64646434633864633465393633663064653139346166393436653938363931663435366435343665 22 | 3362313635653035343865363033323762383535613130300a326563633934316561656362633665 23 | 35396630316530353764616338353436323832616365383731303165626535306132353663336465 24 | 6564373632303764370a643631303336653433623531643565306139383335366262303064623263 25 | 3664 26 | ... 27 | """ 28 | yaml_content_expected = """--- 29 | Neverever: gonna 30 | give: you 31 | up: !vault | 32 | $ANSIBLE_VAULT;1.1;AES256 33 | 64646434633864633465393633663064653139346166393436653938363931663435366435343665 34 | 3362313635653035343865363033323762383535613130300a326563633934316561656362633665 35 | 35396630316530353764616338353436323832616365383731303165626535306132353663336465 36 | 6564373632303764370a643631303336653433623531643565306139383335366262303064623263 37 | 3664 38 | 39 | ... 40 | """ 41 | 42 | with open("test1_password.txt", "w", encoding="utf-8") as f: 43 | f.write(vault_pass) 44 | with open("test1_initial.yaml", "w", encoding="utf-8") as f: 45 | f.write(yaml_content_initial) 46 | with open("test1_expect.yaml", "w", encoding="utf-8") as f: 47 | f.write(yaml_content_expected) 48 | 49 | # Run vaulti, editing with sed 50 | session.env["VISUAL"] = "sed -i s/Never/Neverever/" 51 | session.env["ANSIBLE_VAULT_PASSWORD_FILE"] = "test1_password.txt" 52 | session.run("bash", "-c", f"vaulti test1_initial.yaml", external=True) 53 | 54 | # Diff them and fail if they are not the same 55 | if filecmp.cmp("test1_initial.yaml", "test1_expect.yaml"): 56 | os.remove("test1_initial.yaml") 57 | os.remove("test1_expect.yaml") 58 | os.remove("test1_password.txt") 59 | else: 60 | raise AssertionError("File did not get changed as expected") 61 | 62 | 63 | @nox.session(name="Change encrypted content?") 64 | def test_print_decrypted_content(session): 65 | 66 | vault_pass = "default" 67 | yaml_content_initial = """--- 68 | Never: gonna 69 | give: you 70 | up: !vault | 71 | $ANSIBLE_VAULT;1.1;AES256 72 | 64646434633864633465393633663064653139346166393436653938363931663435366435343665 73 | 3362313635653035343865363033323762383535613130300a326563633934316561656362633665 74 | 35396630316530353764616338353436323832616365383731303165626535306132353663336465 75 | 6564373632303764370a643631303336653433623531643565306139383335366262303064623263 76 | 3664 77 | ... 78 | """ 79 | yaml_content_expected = """--- 80 | Never: gonna 81 | give: you 82 | up: !ENCRYPT lol_changed 83 | ... 84 | """ 85 | 86 | with open("test2_password.txt", "w", encoding="utf-8") as f: 87 | f.write(vault_pass) 88 | with open("test2_initial.yaml", "w", encoding="utf-8") as f: 89 | f.write(yaml_content_initial) 90 | with open("test2_expect.yaml", "w", encoding="utf-8") as f: 91 | f.write(yaml_content_expected) 92 | 93 | # Run vaulti, editing with sed 94 | session.env["VISUAL"] = "sed -i s/lol/lol_changed/" 95 | session.env["ANSIBLE_VAULT_PASSWORD_FILE"] = "test2_password.txt" 96 | session.run("bash", "-c", f"vaulti test2_initial.yaml", external=True) 97 | 98 | # Command to print the decrypted and hopefully changed content 99 | session.run("bash", "-c", f"vaulti -r test2_initial.yaml > test2_decrypted.yaml", external=True) 100 | 101 | # Diff them and fail if they are not the same 102 | if filecmp.cmp("test2_decrypted.yaml", "test2_expect.yaml"): 103 | os.remove("test2_initial.yaml") 104 | os.remove("test2_expect.yaml") 105 | os.remove("test2_decrypted.yaml") 106 | os.remove("test2_password.txt") 107 | else: 108 | raise AssertionError("File did not get decrypted as expected") 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Vaulti 2 | 3 | Like `ansible-vault edit`, but for files with inline encrypted variables! 4 | 5 | Edit, create, encrypt and decrypt ansible-vault in-line encrypted variables in yaml files. 6 | 7 | 8 | ## Usage example 9 | 10 | https://github.com/user-attachments/assets/74480694-3333-4405-8f68-248af21c9999 11 | 12 | 13 | ## Quick start 14 | 15 | ```shell 16 | # Install it 17 | pip install vaulti-ansible 18 | 19 | # Open a file for editing 20 | vaulti file.yml 21 | 22 | # See more options 23 | vaulti -h 24 | ``` 25 | 26 | ## Features 27 | 28 | - Encrypt or decrypt files by adding or removing custom tags 29 | - Change encrypted values directly 30 | - Works for simple variables, lists, dicts, multiline variables (literals) and anchors 31 | - Works with multiple / non-default vault-ids 32 | - Print decrypted files to stdout 33 | - Some Quality of Life features to let you reopen the file if simple mistakes are made 34 | 35 | See [docs folder](docs/) for more examples and user guides 36 | 37 | 38 | ## Why this exists 39 | 40 | The standard `ansible-vault` works fine for encrypting/decrypting/editing whole files, but there are times you don't want to encrypt entire files; for example: 41 | 42 | If you use AWX/AAP, having vault-encrypted files is a bit difficult; you either have to: 43 | 44 | - include the vault password in whichever container/Execution environment you are running the playbooks (therefore requiring a custom container image), or 45 | - decrypt the file when syncing the inventory (making all your secrets plaintext for those with high enough access in AWX) 46 | 47 | Additionally, if your control repo is getting large, with lots of `host_vars` variables, `group_vars` variables, complex playbooks and roles, and you want 48 | to find out where certain variables are defined, you won't be able to search full vault-encrypted files easily, since all the keys are also encrypted. 49 | 50 | So then you try inline encryption, which solves pretty much all of these problems, but using it with `ansible-vault edit ` is no longer possible... 51 | you now have to do something like this instead: 52 | 53 | ```shell 54 | ## To encrypt: 55 | ansible-vault encrypt_string # 56 | SomePasswordOrSomething # , NOT unless you need the newline encrypted too 57 | # Then copy the output into your yaml file, making sure the indentation is still ok 58 | 59 | ## To edit: 60 | # Not possible, just encrypt a new string and replace it. 61 | 62 | ## To view: 63 | ansible -i the/relevant/inventory the-relevant-host -m debug -a "var=TheRelevantVariable" 64 | 65 | ## To decrypt: 66 | # Not possible, just view it and copy-paste the content where needed 67 | 68 | ``` 69 | 70 | Not really easy to remember the encrypt and view steps, pretty error-prone and requires you to actually run something with ansible, putting the variable 71 | somewhere where it will actually be read (hostvars or groupvars). It is *much* easier to just open the decrypted content and edit it directly. 72 | 73 | 74 | ## Caveats 75 | 76 | Editing yaml files using any YAML-loader is a "destructive" process. The content is loaded, and then 77 | dumped/re-generated as a new file after editing. Anything that isn't loaded, like comments or extra 78 | newlines will be gone. 79 | 80 | Since I'm using `ruamel.yaml`, it stores the comments and newlines too, however it will still 81 | re-generate the content every time you use it. This results in some "non-negotiable" changes to your 82 | files that you should be aware of: 83 | 84 | - Indentation for your multiline strings will always end up with a fixed (default two) spaces relative to the variable it belongs to; 85 | i.e. not the 10 spaces indented or whatever the default is from the `ansible-vault encrypt_string` output. This is good for consistency, but it does mean that the indentation 86 | of your inline-encrypted variables will probably change the first time you use this if you've previously used `ansible-vault encrypt_string` to generate the encrypted strings. 87 | If you don't change the decrypted value it should remain the same though, except for the indent change. 88 | - Extra whitespaces will be removed whereever it is found (for example `key: value` -> `key: value`) 89 | 90 | Also, there are a few "opinionated" things I've hardcoded, which are relatively easy to comment out or change in `setup_yaml()` if you wish it. 91 | I might make this configurable via a config file later if there is a need. 92 | 93 | - Header (`---`) and footer (`...`) will be added automatically to the variable file if it doesn't exist. 94 | - An indent equals two spaces 95 | - The hyphen starting each list items is indented, not inline with the parent key 96 | - An extra newline is added below the ansible-vault output, for readability. 97 | - No automatic line breaks for long values. 98 | 99 | Finally, a word on diffs. The utility revolves around decrypting and reencrypting the variables, 100 | which means that every time you open a file with it, the encrypted string actually changes (because 101 | it has a different salt for each reencrypt). Part of the utility is therefore dedicated to looping 102 | through the re-encrypted file, comparing it with the original decrypted data, and preferring the old 103 | encrypted string if the actual decrypted value hasn't changed. That means that any git diff produced 104 | by these changes will usually only involve the relevant changed variables, but it is a "best effort" 105 | process based on very simple list index and dict key lookups. If you change the number encrypted 106 | variables in a list, the items whose list index was changed will be re-encrypted with a new salt, 107 | since the original value cannot be found. Same goes for any variables where you change the key name. 108 | Create the key/entry with a regular editor first if this is important to you. 109 | 110 | ## Dependencies 111 | 112 | Won't put the dependencies in the `pyproject.toml` file for now, since with Ansible, sometimes you 113 | want `ansible-core` on a specific version to keep a consistent execution environment. Any mention of 114 | the required libraries will make pip upgrade `ansible` and `ansible-core` packages even if the 115 | requirements don't make it necessary. 116 | 117 | Having to use `--no-deps` for installing this tool in a non-breaking way is just asking for trouble. 118 | 119 | Dependencies and their reasons are: 120 | 121 | ``` 122 | # We're using typing classes from Python3.9, and __future__ annotations is not not available before python 3.7. 123 | # Since ansible seems to skip 3.7, let's just say 3.8. Might work with 3.7, if you are using that for some reason. 124 | python>=3.8 125 | 126 | ruamel.yaml>=0.16.6 # Won't work before this version due to TaggedScalar changes 127 | ansible>=2.4.0.0 # Won't work for older versions due to pycrypto 128 | ``` 129 | -------------------------------------------------------------------------------- /src/vaulti_ansible/vaulti.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Utility to edit yaml files with inline ansible vault variables 5 | 6 | This is a utility to make ansible-vault inline encrypted variables a billion 7 | times easier to work with. Useful if you want to store the variables safely in 8 | AWX, or want to avoid encrypting entire files because you want to be able to 9 | search for all of your variables, but you don't like the way you currently have 10 | to read and write the variables. 11 | 12 | It opens an editor based on your EDITOR environment variable, where the 13 | variables have been decrypted. This is indicated by a tag, which is set to 14 | "!ENCRYPTED" by default. 15 | 16 | If it cannot decrypt the variable for any reason, it will be indicated with a 17 | "!VAULT_INVALID" tag, which will be translated back to its original value when 18 | you close the editor. It will still try to reencrypt. 19 | 20 | From here, you can add or remove the tags from whichever variables you want, 21 | and when you save and quit, it will reencrypt and decrypt things for you as you 22 | specified. 23 | 24 | Since ruamel.yaml does a lot of stuff in the background, there are some things 25 | that will be changed automatically: 26 | - Indentation for your multiline strings will always end up with a fixed 27 | (default 2) spaces relative to the variable it belongs to; i.e. not the 10 28 | spaces indented or whatever the default is from the `ansible-vault 29 | encrypt_string` output. 30 | - Header (---) and footer (...) will be added automatically to the variable 31 | file if it doesn't exist 32 | - Extra whitespaces will be removed (for example `key: value` -> `key: value`.) 33 | - An extra newline is added below the ansible-vault output, for readability. 34 | 35 | This script is developed by someone who just wants it to work, so feel free to 36 | change it if you want to make it work better. 37 | 38 | Usage: 39 | 40 | ./vaulti ... 41 | ./vaulti --ask-vault-pass 42 | ./vaulti --vault-id myid1@prompt --vault-id myid2@passwordfile 43 | ./vaulti -h 44 | 45 | 46 | Contributions: 47 | 48 | Thanks to Nebucatnetzer for refactoring! 49 | 50 | """ 51 | 52 | from __future__ import annotations 53 | 54 | import argparse 55 | import logging 56 | import os 57 | import subprocess 58 | import sys 59 | import tempfile 60 | import re 61 | 62 | from argparse import Namespace 63 | from pathlib import Path 64 | from typing import Any 65 | from typing import BinaryIO 66 | from typing import IO 67 | from typing import Iterable 68 | from typing import Union 69 | from typing import List 70 | 71 | # Since we are not adding requirements to the project (see README for details), we'll try to import 72 | # ruamel.yaml and ansible early, catching exceptions and giving a friendly explanation 73 | #pylint: disable-msg=C0103 74 | modules_not_found: bool = False 75 | try: 76 | import ruamel.yaml as test_r # pylint: disable=unused-import 77 | except ModuleNotFoundError: 78 | print( 79 | "Vaulti requires the ruamel.yaml module to work (version 0.16.6 or higher). " 80 | "Please install it, for example with 'pip install ruamel.yaml'", 81 | file=sys.stderr 82 | ) 83 | modules_not_found = True 84 | 85 | try: 86 | import ansible as test_a # pylint: disable=unused-import 87 | except ModuleNotFoundError: 88 | print( 89 | "Vaulti requires the ansible-core module to work (ansible version 2.4.0.0 or higher).\n" 90 | "It isn't installed automatically since we often want to stay on a specific ansible " 91 | "version in production environments.\n" 92 | "Please install it youself, for example with 'pip install ansible'", 93 | file=sys.stderr 94 | ) 95 | modules_not_found = True 96 | 97 | if modules_not_found: 98 | sys.exit(1) 99 | #pylint: enable-msg=C0103 100 | 101 | # Disable import position check for the block due to the above try-except 102 | #pylint: disable=wrong-import-position 103 | from ansible import constants as C 104 | from ansible.cli import CLI 105 | from ansible.errors import AnsibleError 106 | from ansible.parsing.dataloader import DataLoader 107 | from ansible.parsing.vault import AnsibleVaultError 108 | from ansible.parsing.vault import (VaultLib, VaultSecret) 109 | 110 | from ruamel.yaml import ScalarNode 111 | from ruamel.yaml import YAML 112 | from ruamel.yaml.comments import ( 113 | CommentedBase, 114 | CommentedMap, 115 | CommentedSeq, 116 | TaggedScalar, 117 | ) 118 | from ruamel.yaml.compat import StringIO 119 | from ruamel.yaml.constructor import (RoundTripConstructor, DuplicateKeyError, ConstructorError) 120 | from ruamel.yaml.error import StringMark # To be able to insert newlines where needed 121 | from ruamel.yaml.parser import ParserError 122 | from ruamel.yaml.scanner import ScannerError 123 | from ruamel.yaml.tokens import ( 124 | CommentToken, 125 | ) # To be able to insert newlines where needed 126 | 127 | from vaulti_ansible.__about__ import __version__ 128 | #pylint: enable=wrong-import-position 129 | 130 | # Definitions for the temporary tag names 131 | # should be descriptive enough to indicate the problem 132 | TAG_NAME_DECRYPTED_SUCCESS = "!ENCRYPT" 133 | TAG_NAME_COULD_NOT_DECRYPT = "!COULD_NOT_DECRYPT" 134 | TAG_NAME_INVALID_VAULT_FORMAT = "!VAULT_FORMAT_ERROR" 135 | TAG_NAME_UNKNOWN_LABEL = "!UNKNOWN_VAULT_ID_LABEL" 136 | TAG_SEPARATOR_VAULTID = ":" # The symbol to denote the ansible-vault id 137 | 138 | StreamType = Union[BinaryIO, IO[str], StringIO] 139 | VAULT = None 140 | 141 | def setup_vault(ask_vault_pass: bool, vault_password_file: str = None, 142 | vault_ids: list = None) -> VaultLib: 143 | """Set up the vault object to use for encryption/decyption. Handles prompting, etc. """ 144 | loader = DataLoader() 145 | logger = logging.getLogger("Vaulti") 146 | 147 | # If custom vault ids are specified (as parameters on the command line), use them 148 | if len(vault_ids) > 0: 149 | vault_ids_final = vault_ids 150 | 151 | # if not, just go with the default (based on environment variables and ansible.cfg) 152 | else: 153 | # This variable might exist, depending on the ansible configuration. Ignore it with pylint 154 | vault_ids_final = C.DEFAULT_VAULT_IDENTITY_LIST # pylint: disable=no-member 155 | 156 | # If a vault password file is specified, add it to the default id 157 | if vault_password_file: 158 | logger.info("Vault password file specified as parameter, adding it as the default vault id") 159 | vault_ids_final.append(f"@{vault_password_file}") 160 | 161 | # Set up vault 162 | try: 163 | vault_secret = CLI.setup_vault_secrets( 164 | loader=loader, 165 | vault_ids=vault_ids_final, 166 | ask_vault_pass=ask_vault_pass, 167 | ) 168 | except AnsibleError as err: 169 | print(f"Could not decrypt. Error is:\n{err}", file=sys.stderr) 170 | # Most likely issue 171 | print("Make sure you point to a valid file if you are using the " 172 | "$ANSIBLE_VAULT_PASSWORD_FILE environment variable", file=sys.stderr) 173 | sys.exit(1) 174 | return VaultLib(vault_secret) 175 | 176 | def get_secret_for_vault_id(vault_lib: VaultLib, vault_id: str) -> VaultSecret: 177 | """ Retrieves the VaultSecret associated with a specific vault-id from a VaultLib object. 178 | Useful for when you want to check whether you have actually added the vault-id at all 179 | (as opposed to added it, but typed the wrong password) 180 | """ 181 | # Access the _secrets attribute to get the dictionary of vault secrets 182 | vault_secrets = vault_lib.secrets # This is a list of tuples (vault_id, VaultSecret) 183 | 184 | for id_label, secret in vault_secrets: 185 | if id_label == vault_id: 186 | return secret 187 | 188 | raise ValueError(f"Vault secret with vault-id '{vault_id}' not found.") 189 | 190 | def extract_vault_label(vaulttext: str) -> str: 191 | """Extracts the label from the Vault ID line in the encrypted data. 192 | Returns an empty string if the default vault-id is used""" 193 | first_line = vaulttext.splitlines()[0] 194 | parts = first_line.split(";") 195 | if len(parts) >= 4: 196 | return parts[3] # This is the label 197 | return "" # Return "" if no label is present 198 | 199 | def constructor_tmp_decrypt(_: RoundTripConstructor, node: ScalarNode) -> TaggedScalar: 200 | """Constructor to translate encrypted values to decrypted values when loading yaml 201 | before opening the editor. When encountering issues, it will not decrypt, but 202 | temporarily change the tag to indicate what went wrong. 203 | """ 204 | 205 | logger = logging.getLogger("Vaulti") 206 | label = extract_vault_label(node.value) 207 | 208 | try: 209 | # pylint: disable=possibly-used-before-assignment 210 | decrypted_value = VAULT.decrypt(vaulttext=node.value).decode("utf-8") 211 | 212 | if label != "": 213 | decrypted_tag_with_label = f"{TAG_NAME_DECRYPTED_SUCCESS}{TAG_SEPARATOR_VAULTID}{label}" 214 | logger.info("Decrypted variable with the vault-id %s", label) 215 | else: 216 | decrypted_tag_with_label = TAG_NAME_DECRYPTED_SUCCESS 217 | logger.info("Decrypted variable with the default vault-id") 218 | 219 | # Make it easier to read decrypted variables with newlines in it 220 | if "\n" in decrypted_value: 221 | logger.info("Printing multiline variable with newlines for easy reading/editing") 222 | taggedscalar = TaggedScalar( 223 | value=decrypted_value, style="|", tag=decrypted_tag_with_label 224 | ) 225 | taggedscalar.yaml_set_anchor(node.anchor) 226 | return taggedscalar 227 | 228 | logger.info("Decrypted variable with the default vault-id") 229 | taggedscalar = TaggedScalar(value=decrypted_value, style="", tag=decrypted_tag_with_label) 230 | taggedscalar.yaml_set_anchor(node.anchor) 231 | return taggedscalar 232 | 233 | except AnsibleVaultError: 234 | # If there is no label, it is probably just a variable encrypted with the wrong key 235 | 236 | if label == "": 237 | logger.info("Could not decrypt variable with default vault id") 238 | taggedscalar = TaggedScalar(value=node.value, style="|", tag=TAG_NAME_COULD_NOT_DECRYPT) 239 | taggedscalar.yaml_set_anchor(node.anchor) 240 | return taggedscalar 241 | # If there is a label, it is probably because you just forgot to load the vault id, 242 | # and the temporary tag name might give you a hint 243 | 244 | try: 245 | # Try this and see if it fails, we dont care about what it returns 246 | get_secret_for_vault_id(VAULT, label) 247 | # If you did load the vault-id and it still failed, it is probably 248 | # just the wrong password 249 | logger.info("Could not decrypt variable") 250 | taggedscalar = TaggedScalar(value=node.value, style="|", tag=TAG_NAME_COULD_NOT_DECRYPT) 251 | taggedscalar.yaml_set_anchor(node.anchor) 252 | return taggedscalar 253 | except ValueError: 254 | # If you did not load it, mention that using the tag 255 | logger.info("Could not decrypt variable, vault-id %s not loaded", label) 256 | taggedscalar = TaggedScalar( 257 | value=node.value, style="|", 258 | tag=f"{TAG_NAME_UNKNOWN_LABEL}{TAG_SEPARATOR_VAULTID}{label}" 259 | ) 260 | taggedscalar.yaml_set_anchor(node.anchor) 261 | return taggedscalar 262 | 263 | except AnsibleError: 264 | # If the format is wrong, add that as a separate tag 265 | logger.info("Could not decrypt variable with invalid format") 266 | taggedscalar = TaggedScalar(value=node.value, style="|", tag=TAG_NAME_INVALID_VAULT_FORMAT) 267 | taggedscalar.yaml_set_anchor(node.anchor) 268 | return taggedscalar 269 | 270 | 271 | 272 | def constructor_tmp_encrypt( 273 | _: RoundTripConstructor, node: ScalarNode, tag_suffix: str = None 274 | ) -> TaggedScalar: 275 | """Constructor to reencrypt values. Will look for vault-id labels in the tag, otherwise 276 | just uses the default vault-id to encrypt. 277 | """ 278 | 279 | if tag_suffix is None: 280 | secret = get_secret_for_vault_id(VAULT, "default") 281 | vault_id = "default" 282 | else: 283 | secret = get_secret_for_vault_id(VAULT, tag_suffix) 284 | vault_id = tag_suffix 285 | 286 | # Seems to need explicit values for secret and vault_id even when you just want the default, 287 | # It seems to just select the first VaultSecret object otherwise, which is rarely default. 288 | encrypted_value = VAULT.encrypt( 289 | plaintext=node.value, secret=secret, vault_id=vault_id 290 | ).decode("utf-8") 291 | 292 | taggedscalar = TaggedScalar(value=encrypted_value, style="|", tag="!vault") 293 | taggedscalar.yaml_set_anchor(node.anchor) 294 | return taggedscalar 295 | 296 | 297 | def constructor_tmp_invalid(_: RoundTripConstructor, node: ScalarNode) -> TaggedScalar: 298 | """ The invalid tag should just be translated directly back to the original tag and value. 299 | Useful for when we are using custom tags, or when you don't want the !vault value to be changed 300 | """ 301 | taggedscalar = TaggedScalar(value=node.value, style="|", tag="!vault") 302 | taggedscalar.yaml_set_anchor(node.anchor) 303 | return taggedscalar 304 | 305 | 306 | 307 | def is_commented_map(data: Any) -> bool: 308 | """Helper function for readability""" 309 | return isinstance(data, CommentedMap) 310 | 311 | 312 | def is_commented_seq(data: Any) -> bool: 313 | """Helper function for readability""" 314 | return isinstance(data, CommentedSeq) 315 | 316 | 317 | def is_tagged_scalar(data: Any) -> bool: 318 | """Helper function for readability""" 319 | return isinstance(data, TaggedScalar) 320 | 321 | 322 | def _process_commented_map( 323 | original_data: CommentedMap, reencrypted_data: CommentedMap 324 | ) -> tuple[CommentedMap, CommentedMap]: 325 | """Helper function for compare_and_update. Loops over keys in a dict""" 326 | for key in reencrypted_data: 327 | if ( 328 | is_tagged_scalar(reencrypted_data[key]) 329 | and reencrypted_data[key].tag.value == "!vault" 330 | ): 331 | ensure_newline(reencrypted_data, key) 332 | if key in original_data: 333 | # If ansible vault fails, use the new data instead of crashing 334 | try: 335 | reencrypted_data[key] = compare_and_update( 336 | original_data=original_data[key], 337 | reencrypted_data=reencrypted_data[key], 338 | ) 339 | except (AnsibleError, AnsibleVaultError): 340 | reencrypted_data[key] = reencrypted_data[key] 341 | return original_data, reencrypted_data 342 | 343 | 344 | def _process_commented_seq( 345 | original_data: CommentedSeq, reencrypted_data: CommentedSeq 346 | ) -> tuple[CommentedSeq, CommentedSeq]: 347 | """Helper function for compare_and_update. Loops over items in a list""" 348 | # Won't use enumerate for now, doesn't seem to be too easy to implement in the middle of this 349 | # recursive logic 350 | for i in range(len(reencrypted_data)): # pylint: disable=consider-using-enumerate 351 | if ( 352 | is_tagged_scalar(reencrypted_data[i]) 353 | and reencrypted_data[i].tag.value == "!vault" 354 | ): 355 | ensure_newline(reencrypted_data, str(i)) 356 | # If ansible vault fails, use the new data instead of crashing 357 | try: 358 | if i < len(original_data): 359 | reencrypted_data[i] = compare_and_update( 360 | original_data=original_data[i], 361 | reencrypted_data=reencrypted_data[i], 362 | ) 363 | else: 364 | reencrypted_data[i] = reencrypted_data[i] 365 | except (AnsibleError, AnsibleVaultError): 366 | reencrypted_data[i] = reencrypted_data[i] 367 | 368 | return original_data, reencrypted_data 369 | 370 | 371 | def compare_and_update( 372 | original_data: Union[CommentedMap | CommentedSeq | TaggedScalar], 373 | reencrypted_data: Union[CommentedMap | CommentedSeq | TaggedScalar], 374 | ) -> Union[CommentedMap | CommentedSeq | TaggedScalar]: 375 | """Take the new and original data, find each !vault entry, and if it exists in the original 376 | data, decrypt both and compare them. If they are the same, prefer the original data, to prevent 377 | useless diffs. Will consider values different if the new and old values have different vault-id 378 | labels. Will also ensure that there is a newline after a vaulted variable (for readability) 379 | """ 380 | 381 | # Loop recursively through everything 382 | if is_commented_map(original_data) and is_commented_map(reencrypted_data): 383 | original_data, reencrypted_data = _process_commented_map( 384 | original_data, reencrypted_data # type: ignore[arg-type] 385 | ) 386 | elif is_commented_seq(original_data) and is_commented_seq(reencrypted_data): 387 | original_data, reencrypted_data = _process_commented_seq( 388 | original_data, reencrypted_data # type: ignore[arg-type] 389 | ) 390 | 391 | elif ( 392 | is_tagged_scalar(original_data) 393 | and original_data.tag.value == "!vault" 394 | and is_tagged_scalar(reencrypted_data) 395 | and reencrypted_data.tag.value == "!vault" 396 | ): 397 | if ( 398 | VAULT.decrypt(original_data.value) == VAULT.decrypt(reencrypted_data.value) and 399 | extract_vault_label(original_data.value) == extract_vault_label(reencrypted_data.value) 400 | ): 401 | return original_data 402 | 403 | return reencrypted_data 404 | 405 | 406 | def ensure_newline(data: Union[CommentedMap, CommentedSeq], key: "str") -> None: 407 | """Utility script, to avoid having to write it twice in the recursive _process_commented*() 408 | functions above""" 409 | comment_nextline = data.ca.items.get(key) 410 | # Ensure that there is at least one newline after the vaulted value, for readability 411 | if comment_nextline is None: 412 | data.ca.items[key] = [None, None, None, None] 413 | # All this just to make a newline... not 100% sure how this StringMark 414 | # stuff works 415 | newline_token = CommentToken( 416 | "\n", 417 | start_mark=StringMark( 418 | buffer=data, pointer=0, name=None, index=0, line=0, column=0 419 | ), 420 | end_mark=StringMark( 421 | buffer=data, pointer=1, name=None, index=1, line=0, column=1 422 | ), 423 | ) 424 | data.ca.items[key][2] = newline_token 425 | 426 | 427 | def yaml_set_anchor(self, value, always_dump=True): 428 | """ Overrides the default yaml_set_anchor function to set it to always_dump=True instead of 429 | false. This is done so that any unused anchors that are loaded are preserved, not removed. 430 | """ 431 | self.anchor.value = value 432 | self.anchor.always_dump = always_dump 433 | 434 | def setup_yaml() -> YAML: 435 | """Set up the neccesary yaml loader stuff""" 436 | yaml = YAML() 437 | # Don't strip out unneccesary quotes around scalar variables 438 | yaml.preserve_quotes = True 439 | # Prevent the yaml dumper from line-breaking the longer variables 440 | yaml.width = 2147483647 441 | # Add --- at the start of the file 442 | yaml.explicit_start = True 443 | # Add ... at the end of the file 444 | yaml.explicit_end = True 445 | # Ensure list items are indented by two, but not inline with the parent variable 446 | yaml.indent(mapping=2, sequence=4, offset=2) 447 | # Override the yaml_set_anchor to keep anchors even if they are unused 448 | CommentedBase.yaml_set_anchor = yaml_set_anchor 449 | 450 | return yaml 451 | 452 | 453 | def read_encrypted_yaml_file(file: Path) -> Any: 454 | """Add a custom constructor to decrypt vault, and load the content. Used for the initial 455 | decryption of the file""" 456 | yaml = setup_yaml() 457 | yaml.constructor.add_constructor("!vault", constructor_tmp_decrypt) 458 | with open(file, "r", encoding="utf-8") as file_to_decrypt: 459 | return yaml.load(file_to_decrypt) 460 | 461 | 462 | def read_yaml_file(file: Path) -> Any: 463 | """Load the yaml file, mainly just to verify that this is a file, and that it is a valid yaml 464 | file 465 | """ 466 | yaml = setup_yaml() 467 | try: 468 | with open(file, "r", encoding="utf-8") as file_to_read: 469 | return yaml.load(file_to_read) 470 | except IsADirectoryError as err: 471 | print(f"Specified file is a directory. Error is:\n{err}", file=sys.stderr) 472 | sys.exit(1) 473 | 474 | 475 | def display_yaml_data(yaml_data: Union[Path, StreamType]) -> None: 476 | """Dumps the unencrypted content to stdout without opening an editor. Useful when you want to 477 | pipe it to something else, use it for git textconv, etc.""" 478 | yaml = setup_yaml() 479 | yaml.dump(data=yaml_data, stream=sys.stdout) 480 | 481 | 482 | def _get_default_editor() -> List[str]: 483 | """Get the default editor from env variables, using sane defaults if the env is not set 484 | """ 485 | editor = os.environ.get("VISUAL", "").split() 486 | if not editor: 487 | editor = os.environ.get("EDITOR", "nano").split() 488 | 489 | if not editor: 490 | print("No editor configured in neither VISUAL nor EDITOR variable. Exiting.") 491 | sys.exit(1) 492 | return editor 493 | 494 | 495 | def open_file_in_default_editor(file_name: Path) -> None: 496 | """Opens a file in the default editor""" 497 | logger = logging.getLogger("Vaulti") 498 | editor = _get_default_editor() 499 | logger.info("Opening editor with params: %s", str(editor)) 500 | subprocess.run(editor + [file_name], check=True) 501 | 502 | 503 | def write_data_to_temporary_file(data_to_write: Union[Path, StreamType]) -> Path: 504 | """Write the yaml contents to a temporary file, for editing""" 505 | yaml = setup_yaml() 506 | # Create a temporary file 507 | with tempfile.NamedTemporaryFile( 508 | mode="w", delete=False, prefix="vaultedit_", suffix=".yaml" 509 | ) as temp_file: 510 | yaml.dump(data_to_write, temp_file) 511 | return Path(temp_file.name) 512 | 513 | def constructor_tmp_encrypt_multi(loader, _tag_suffix: str, node: ScalarNode) -> TaggedScalar: 514 | """Wrapper function for the multiconstructor. The multiconstructor requires a function which 515 | has a parameter that takes the tag_suffix variable. Since we are not using the suffix in this 516 | case, we just pass it to our standard constructor function, dropping the tag_suffix variable""" 517 | return constructor_tmp_encrypt(loader, node, tag_suffix=_tag_suffix) 518 | 519 | def constructor_tmp_invalid_multi(loader, _tag_suffix: str, node: ScalarNode) -> TaggedScalar: 520 | """Wrapper function for the multiconstructor. The multiconstructor requires a function which 521 | has a parameter that takes the tag_suffix variable. Since we are not using the suffix in this 522 | case, we just pass it to our standard constructor function, dropping the tag_suffix variable""" 523 | return constructor_tmp_invalid(loader, node) 524 | 525 | def encrypt_and_write_tmp_file( 526 | tmp_file: Path, final_file: Path, original_data: CommentedMap 527 | ) -> None: 528 | """Reencrypts yaml data and writes it to a file using lots of custom constructors. Also ensures 529 | that the user gets a chance to reopen invalid files, etc.""" 530 | 531 | yaml = setup_yaml() 532 | 533 | 534 | # Register the constructor to let the yaml loader do the 535 | # reencrypting for you Adding it this late to avoid encryption step 536 | # before the editor opens 537 | yaml.constructor.add_multi_constructor( 538 | f"{TAG_NAME_DECRYPTED_SUCCESS}{TAG_SEPARATOR_VAULTID}", 539 | constructor_tmp_encrypt_multi 540 | ) 541 | 542 | # Also register constructors for the temporary tags added in constructor_tmp_decrypt() 543 | # Multiconstructors used for when the tag might includes a vault-id 544 | yaml.constructor.add_multi_constructor( 545 | TAG_NAME_UNKNOWN_LABEL, 546 | constructor_tmp_invalid_multi 547 | ) 548 | yaml.constructor.add_multi_constructor( 549 | TAG_NAME_INVALID_VAULT_FORMAT, 550 | constructor_tmp_invalid_multi 551 | ) 552 | yaml.constructor.add_constructor(TAG_NAME_DECRYPTED_SUCCESS, constructor_tmp_encrypt) 553 | yaml.constructor.add_constructor(TAG_NAME_COULD_NOT_DECRYPT, constructor_tmp_invalid) 554 | 555 | # Not sure why I need to override the constructor for the !vault tag, since this is using 556 | # another yaml object than the one where the constructor was added, , but apparently it is 557 | # needed. Adding it in case someone pastes vault-encrypted variables when in the vaulti editor, 558 | # and we need to prevent the yaml library from decrypting and putting the unencrypted value in 559 | # the final file. 560 | yaml.constructor.add_constructor("!vault", constructor_tmp_invalid) 561 | 562 | def prompt_user_action() -> str: 563 | while True: 564 | try: 565 | user_input = input("Keep editing file (e), Discard changes (d) ? ").strip().lower() 566 | if user_input in ['e', 'd']: 567 | return user_input 568 | print("Invalid input. What would you like to do?") 569 | except KeyboardInterrupt: 570 | sys.exit(0) 571 | 572 | # After the editor is closed, reload the yaml from the tmp-file 573 | # Give the user a chance to re-open the file if the yaml could not be parsed 574 | is_file_parsed = False 575 | while not is_file_parsed: 576 | with open(tmp_file, "r", encoding="utf-8") as file: 577 | try: 578 | edited_data = yaml.load(file) 579 | is_file_parsed = True 580 | except (ScannerError, ParserError, ValueError, 581 | DuplicateKeyError, ConstructorError) as err: 582 | if err is ValueError: 583 | print(f"Encountered Vault ID which has not been loaded. Error is:\n{err}") 584 | else: 585 | print(f"The edited file is no longer valid YAML. Error is:\n{err}") 586 | print("What would you like to do?") 587 | user_retry = prompt_user_action() 588 | 589 | if user_retry == 'e': 590 | open_file_in_default_editor(tmp_file.absolute()) 591 | elif user_retry == 'd': 592 | print("Changes discarded. Exiting") 593 | sys.exit(0) 594 | except AnsibleError as err: 595 | print(f"AnsibleError. Error is:\n{err}", file=sys.stderr) 596 | sys.exit(1) 597 | 598 | # Loop through all the values of the new data, making sure that 599 | # any encrypted data unchanged from the original still uses the 600 | # original vault encrypted data. This makes your git diffs much 601 | # cleaner. 602 | final_data = compare_and_update(original_data, edited_data) 603 | # Then write the final data back to the original file 604 | with open(final_file, "w", encoding="utf-8") as file: 605 | yaml.dump(final_data, file) 606 | 607 | # A common mistake is to comment out a decrypted secret line, 608 | # making it plaintext. Ensure the user is notified of this. 609 | with open(final_file, "r", encoding="utf-8") as file: 610 | content = file.read() 611 | if re.search(r".*#.*" + re.escape(TAG_NAME_DECRYPTED_SUCCESS) + r".*", content): 612 | print( 613 | (f"WARNING! The final file '{final_file}' seems to have secrets that were not " 614 | "reencrypted due to being commented out in the editor! Search the file for " 615 | f"instances of '{TAG_NAME_DECRYPTED_SUCCESS}' that are commented out."), 616 | file=sys.stderr 617 | ) 618 | 619 | 620 | def parse_arguments() -> Namespace: 621 | """Parse the arguments from the command line""" 622 | parser = argparse.ArgumentParser( 623 | prog="vaulti", description="Create and edit ansible-vault inline encrypted variables!" 624 | ) 625 | 626 | parser.add_argument( 627 | "-r", 628 | "--view", 629 | action="store_true", 630 | help="Just print the decrypted output, don't open an editor. " 631 | "WARNING: This will print your secrets in plaintext.", 632 | ) 633 | parser.add_argument( 634 | "-f", 635 | "--force", 636 | "--force-create", 637 | action="store_true", 638 | help="If the file you specified does not already exist, create it instead of erroring.", 639 | ) 640 | 641 | parser.add_argument( 642 | "-v", 643 | "--verbose", 644 | action="store_const", 645 | dest="loglevel", 646 | const=logging.INFO, 647 | help="Print more details, for debugging. " 648 | "WARNING: This will probably print your secrets in plaintext!", 649 | ) 650 | parser.add_argument( 651 | "files", nargs="+", help="Specify one or more files that the script should open. They are " 652 | " processed one by one, so you must save and/or quit the editor in between each one." 653 | ) 654 | parser.add_argument( 655 | '--vault-id', action='append', 656 | help='Specify Vault ID(s) on the format label@sourcefile or label@prompt. ' 657 | "One vault id per '--vault-id' parameter", default=[] 658 | ) 659 | parser.add_argument( 660 | "-J", "--ask-vault-pass", "--ask-vault-password", action="store_true", 661 | help="Get prompted for the default vault-id " 662 | "(basically the same as '--vault-id @prompt'). " 663 | "Only works for variables encrypted with the default vault-id.", 664 | ) 665 | parser.add_argument( 666 | "--vault-password-file", "--vault-pass-file", 667 | help="Specify the password file for the default vault-id " 668 | "(basically the same as '--vault-id @somefile.txt'). " 669 | "Only works for variables encrypted with the default vault-id.", 670 | ) 671 | parser.add_argument( 672 | "-V", "--version", action="version", 673 | version=f"v{__version__}", 674 | help="Print the version of vaulti", 675 | ) 676 | 677 | return parser.parse_args() 678 | 679 | 680 | def main_loop(filenames: Iterable[Path], view_only: bool, force_create: bool) -> None: 681 | """Loop through each file specified as params""" 682 | logger = logging.getLogger("Vaulti") 683 | for filename in filenames: 684 | 685 | logger.info("Starting process for file %s", filename) 686 | 687 | # Has the file been created by this script? 688 | file_force_created = False 689 | 690 | # Read the original file without custom constructors (for comparing 691 | # later) (Deepcopy doesn't seem to work, so just load it before 692 | # defining custom constructors 693 | try: 694 | original_data = read_yaml_file(filename) 695 | except (ScannerError, DuplicateKeyError, ConstructorError) as err: 696 | print(f"'{filename}' is not a valid YAML file. Error is\n{err}", file=sys.stderr) 697 | sys.exit(1) 698 | except FileNotFoundError: 699 | if force_create: 700 | logger.info("--force specified as parameter, creating file %s", filename) 701 | with open(filename, "x", encoding="utf-8") as empty_file: 702 | original_data = empty_file 703 | file_force_created = True 704 | else: 705 | print( 706 | f"File '{filename}' doesn't exist. Create non-existant files with -f.", 707 | file=sys.stderr 708 | ) 709 | sys.exit(1) 710 | 711 | # Load the yaml file into memory (will now auto-decrypt vault because 712 | # of the constructors) 713 | decrypted_data = read_encrypted_yaml_file(filename) 714 | 715 | if view_only: 716 | logger.info("--view specified as parameter, will display and exit") 717 | display_yaml_data(decrypted_data) 718 | continue # Continue to the next loop (file) 719 | 720 | # Run the rest inside a try-finally block to make sure the decrypted 721 | # tmp-file is deleted afterwards 722 | try: 723 | temp_filename = write_data_to_temporary_file(decrypted_data) 724 | logger.info("Created temporary file %s", temp_filename) 725 | created_time = os.stat(temp_filename).st_ctime 726 | open_file_in_default_editor(temp_filename.absolute()) 727 | # Don't do anything if the file hasn't been changed since its creation 728 | changed_time = os.stat(temp_filename).st_ctime 729 | if created_time != changed_time: 730 | logger.info("File was saved after being created, continuing") 731 | encrypt_and_write_tmp_file( 732 | tmp_file=temp_filename, 733 | final_file=filename, 734 | original_data=original_data, 735 | ) 736 | else: 737 | # If the file was created but never changed, delete it 738 | logger.info("File was not saved after being created, exiting") 739 | if file_force_created: 740 | os.unlink(filename) 741 | finally: 742 | logger.info("Remove temporary file %s", temp_filename) 743 | os.unlink(temp_filename) 744 | 745 | 746 | def main() -> None: 747 | """Parse arguments and set up logging before moving on to the main loop""" 748 | args = parse_arguments() 749 | # Need to use the VAULT variable in the custom constructors, and I couldn't figure 750 | # out how to pass it as an extra parameter to the add_constructor function 751 | # pylint: disable=global-statement 752 | global VAULT 753 | 754 | logging.basicConfig(level=args.loglevel, format="%(levelname)s: %(message)s") 755 | logger = logging.getLogger("Vaulti") 756 | logger.info("Initializing vault setup") 757 | 758 | try: 759 | VAULT = setup_vault( 760 | ask_vault_pass=args.ask_vault_pass, 761 | vault_password_file=args.vault_password_file, 762 | vault_ids=args.vault_id 763 | ) 764 | except KeyboardInterrupt: 765 | logger.info("User interrupted process, exiting") 766 | sys.exit(0) 767 | 768 | main_loop(args.files, view_only=args.view, force_create=args.force) 769 | 770 | 771 | if __name__ == "__main__": 772 | main() 773 | --------------------------------------------------------------------------------