├── .github └── workflows │ └── publish.yml ├── .gitignore ├── LICENSE ├── README.md ├── demos ├── decrypt.gif ├── decrypt.tape ├── setup.gif ├── setup.tape ├── updatekeys.gif └── updatekeys.tape ├── github-to-sops ├── github_to_sops └── __init__.py └── pyproject.toml /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python Package 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'pyproject.toml' 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | deploy: 15 | runs-on: ubuntu-latest 16 | environment: release 17 | permissions: 18 | id-token: write 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Set up Python 22 | uses: actions/setup-python@v5 23 | with: 24 | python-version: "3.12" 25 | cache: pip 26 | cache-dependency-path: pyproject.toml 27 | - name: Install dependencies 28 | run: | 29 | pip install setuptools wheel build 30 | - name: Build 31 | run: | 32 | python -m build 33 | - name: Publish 34 | uses: pypa/gh-action-pypi-publish@release/v1 35 | 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | venv 6 | .eggs 7 | .pytest_cache 8 | *.egg-info 9 | .DS_Store 10 | dist 11 | build 12 | .aider* 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Taras Glek 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | github-to-sops integrates SOPS with github team/user identities. Use sops + github instead of having to operate Hashicorp Vault, AWS Secret Manager or just stuffing everything into github action secrets, or fighting with GPG. 2 | 3 | ## Why? 4 | 5 | I think SOPS is the simplest way to manage secrets for team and individual projects, especially when combined with github as a key distribution mechanism. 6 | 7 | This script makes it easy to setup [SOPS](https://github.com/getsops/sops) as a lightweight gitops alternative to AWS Secrets Manager, AWS KMS, Hashicorp Vault. 8 | 9 | SOPS is helpful to avoid the push-and-pray (https://dagger.io/ came up with this term and solution for it) pattern where all secrets for github actions are stored in [Github Secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) such that nobody can repro stuff locally. With sops one can give github actions a single age private key and share all the development keys with rest of team on equal footing with CI/CD env. 10 | 11 | ## Requirements 12 | 13 | * [sops](https://github.com/getsops/sops) 14 | * https://github.com/Mic92/ssh-to-age/ (until the [SOPS ssh backend](https://github.com/getsops/sops/pull/1134) lands). 15 | * Python3 16 | * [pip](https://pip.pypa.io/en/stable/installation/) 17 | 18 | ## Installation 19 | The latest version of github-to-sops can be cloned locally or installed using pip: 20 | ```bash 21 | pip install github-to-sops 22 | ``` 23 | 24 | On Mac or Linux you can install sops, ssh-to-age using: 25 | ```bash 26 | github-to-sops install-binaries 27 | ``` 28 | 29 | ## Implementation 30 | 31 | This generates a nice .sops.yaml file with comments indicating where the keys came from to make key rotation easier. 32 | 33 | Idea for this originated in https://github.com/tarasglek/chatcraft.org/pull/319 after I got sick of devising a secure secret distribution scheme for every small project. 34 | 35 | ## Contributions Welcome 36 | * Tests 37 | * Binary build for python-less environments 38 | * Would be nice to add is ACLs and an integrity check to keys being used. 39 | 40 | ## Examples: 41 | 42 | I wrote an indepth explanation and screencasts on my blog post introducing [github-to-sops](https://taras.glek.net/post/github-to-sops-lighter-weight-secret-management/#heres-how-you-get-started). 43 | 44 | ## Env vars: 45 | 46 | * GITHUB_TOKEN: optional github token which helps avoid rate limiting. 47 | 48 | I tried to make the code work without github tokens, but github requires them for private repos and does aggressive rate-limiting without them. See github docs on how to obtain GITHUB_TOKEN https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens 49 | 50 | 51 | ### Example workflow for secrets with github 52 | 53 | Import all public keys for contributors from an existing github project 54 | ```bash 55 | ./github-to-sops import-keys > .sops.yaml 56 | ``` 57 | of if your repo isn't published to github or you aren't working inside a git checkout 58 | ``` 59 | ./github-to-sops import-keys --github-url https://github.com/tarasglek/chatcraft.org 60 | ``` 61 | lets see 62 | ```bash 63 | cat .sops.yaml 64 | ``` 65 | ```yaml 66 | creation_rules: 67 | - key_groups: 68 | - age: 69 | - age19j4d6v9j7rx5fs629fu387qz4zmlpsqjexa4s08tkfrrmfdl5cwqjlaupd # humphd 70 | - age13runq29jhy9kfpaegczrzttykerswh0qprq59msgd754yermtfmsa3hwg2 # tarasglek 71 | ``` 72 | 73 | Put a sample secret in yaml 74 | 75 | ```bash 76 | echo -e "secrets:\n SECRET_KEY: dontlook" | sops --input-type yaml --output-type yaml -e /dev/stdin > secrets.enc.yaml 77 | ``` 78 | Lets take a peek 79 | ```bash 80 | head -n 9 secrets.enc.yaml 81 | ``` 82 | ```yaml 83 | secrets: 84 | SECRET_KEY: ENC[AES256_GCM,data:MKKR6B0h1iA=,iv:KegjC62NQxich1dtodVF3aVnchf/fB+KQbtETh+4CaY=,tag:2+5mk4YMKKxLqaCOpZVNSA==,type:str] 85 | sops: 86 | kms: [] 87 | gcp_kms: [] 88 | azure_kv: [] 89 | hc_vault: [] 90 | age: 91 | - recipient: age19j4d6v9j7rx5fs629fu387qz4zmlpsqjexa4s08tkfrrmfdl5cwqjlaupd 92 | ``` 93 | ^ is safe to commit! 94 | 95 | #### Decrypting secrets using ssh keys 96 | 97 | Easy way: 98 | 99 | ```bash 100 | github-to-sops sops -d secrets.enc.yaml 101 | ``` 102 | 103 | More complicated details: 104 | 105 | ```bash 106 | export SOPS_AGE_KEY=$(ssh-to-age -private-key < ~/.ssh/id_ed25519) 107 | ``` 108 | 109 | Lets extract our secret in a way that's useful for automation 110 | ```bash 111 | sops --extract '["secrets"]["SECRET_KEY"]' -d secrets.env.yaml 112 | ``` 113 | ``` 114 | dontlook 115 | ``` 116 | 117 | `sops -i secrets.env.yaml` is useful for interactive editing. 118 | 119 | #### Bulk-updating secrets+keys when someone is added/removed from project 120 | 121 | ```bash 122 | github-to-sops refresh-secrets 123 | ``` 124 | 125 | ## Usage: 126 | ``` 127 | ./github-to-sops -h 128 | usage: github-to-sops [-h] {import-keys,refresh-secrets,sops} ... 129 | 130 | Manage GitHub SSH keys and generate SOPS-compatible SSH key files. 131 | 132 | options: 133 | -h, --help show this help message and exit 134 | 135 | Commands: 136 | {import-keys,refresh-secrets,sops} 137 | sops Run sops with SOPS_AGE_KEY set from ~/.ssh/id_ed25519 138 | import-keys Import SSH keys of GitHub repository contributors or specified github users and output that info into a useful format like sops or ssh authorized_keys 139 | refresh-secrets Find all .sops.yaml files in the repo that are managed by git and run `import-keys --inplace-edit .sops.yaml` on them. 140 | 141 | Example invocations: 142 | - `./github-to-sops import-keys --github-url https://github.com/tarasglek/chatcraft.org --key-types ssh-ed25519 --format sops` 143 | - `./github-to-sops import-keys --github-url https://github.com/tarasglek/chatcraft.org --format authorized_keys` 144 | - `./github-to-sops import-keys --local-github-checkout . --format sops --known-hosts ~/.ssh/known_hosts --key-types ssh-ed25519` 145 | - `./github-to-sops refresh-secrets` 146 | ``` 147 | -------------------------------------------------------------------------------- /demos/decrypt.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tarasglek/github-to-sops/382512a0a051ebd3c60f0d82b596bb3632c7b408/demos/decrypt.gif -------------------------------------------------------------------------------- /demos/decrypt.tape: -------------------------------------------------------------------------------- 1 | Output decrypt.gif 2 | 3 | Require ssh-to-age 4 | Require sops 5 | 6 | Set Shell "bash" 7 | Set FontSize 14 8 | Set Width 1200 9 | Set Height 300 10 | Set Margin 0 11 | Set BorderRadius 0 12 | Set TypingSpeed 50ms 13 | Set WindowBarSize 0 14 | 15 | Type "export SOPS_AGE_KEY=$(ssh-to-age -private-key < ~/.ssh/id_ed25519)" 16 | Enter 17 | Sleep 4s 18 | Type "sops -d keys.enc.yaml" 19 | Enter 20 | Sleep 10s 21 | -------------------------------------------------------------------------------- /demos/setup.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tarasglek/github-to-sops/382512a0a051ebd3c60f0d82b596bb3632c7b408/demos/setup.gif -------------------------------------------------------------------------------- /demos/setup.tape: -------------------------------------------------------------------------------- 1 | Output setup.gif 2 | 3 | Require github-to-sops 4 | Require yq 5 | Require git 6 | 7 | Set Shell "bash" 8 | Set FontSize 14 9 | Set Width 1200 10 | Set Height 600 11 | Set Margin 0 12 | # Set TypingSpeed 50ms 13 | 14 | Type "yq . < keys.unsafe.yaml" 15 | Enter 16 | Sleep 4s 17 | Type "github-to-sops --github-url https://github.com/tarasglek/chatcraft.org --key-types ssh-ed25519 --format sops > .sops.yaml" 18 | Enter 19 | Sleep 10s 20 | Type "yq . < .sops.yaml" 21 | Enter 22 | Sleep 10s 23 | Type "sops -e keys.unsafe.yaml > keys.enc.yaml" 24 | Enter 25 | Sleep 4s 26 | Type "head -n 15 keys.enc.yaml | yq ." 27 | Enter 28 | Sleep 10s 29 | Type "git add keys.enc.yaml" 30 | Enter 31 | Sleep 10s 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /demos/updatekeys.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tarasglek/github-to-sops/382512a0a051ebd3c60f0d82b596bb3632c7b408/demos/updatekeys.gif -------------------------------------------------------------------------------- /demos/updatekeys.tape: -------------------------------------------------------------------------------- 1 | Output updatekeys.gif 2 | 3 | Require ssh-to-age 4 | Require sops 5 | Require github-to-sops 6 | 7 | Set Shell "bash" 8 | Set FontSize 14 9 | Set Width 1200 10 | Set Height 600 11 | Set Margin 0 12 | Set BorderRadius 0 13 | Set WindowBarSize 0 14 | 15 | 16 | Type "github-to-sops --inplace-edit .sops.yaml --github-users tarasglek,rjwignar" 17 | Enter 18 | Sleep 4s 19 | Type "yq . < .sops.yaml" 20 | Enter 21 | Sleep 10s 22 | Type "export SOPS_AGE_KEY=$(ssh-to-age -private-key < ~/.ssh/id_ed25519)" 23 | Enter 24 | Sleep 4s 25 | Type "sops updatekeys -y keys.enc.yaml" 26 | Enter 27 | Sleep 10s 28 | -------------------------------------------------------------------------------- /github-to-sops: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ./github_to_sops/__init__.py $@ 3 | -------------------------------------------------------------------------------- /github_to_sops/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Standalone (no-dependencies beyond Python) script fetches SSH keys of GitHub repository contributors and generates SOPS-compatible SSH key files. 4 | """ 5 | 6 | import argparse 7 | import json 8 | import logging 9 | import os 10 | import sys 11 | import subprocess 12 | from typing import Any, Dict, Optional, List, Set, TextIO 13 | from urllib import request, error 14 | import re 15 | 16 | GITHUB_TO_SOPS_TAG = "https://github.com/tarasglek/github-to-sops" 17 | GITHUB_API_BASE_URL = "api.github.com/repos" 18 | GENERATED_MSG = ( 19 | f"Generated by `{' '.join(sys.argv)}` {GITHUB_TO_SOPS_TAG}" 20 | ) 21 | 22 | SOPS_TEMPLATE = f""" 23 | creation_rules: 24 | - key_groups: 25 | - age: 26 | - Mark stuff to replace by having a line with {GITHUB_TO_SOPS_TAG} which can be within a comment or not 27 | - Following lines with same indent get dropped 28 | # EOF 29 | """ 30 | 31 | def process_template(template, tag, output_fd): 32 | """ 33 | 1. Takes a template string 34 | 2. Finds line containing the specified tag 35 | 3. Detects whitespace on the tag line and stores that as line_prefix 36 | 4. yields it 37 | 5. Finds first line after the tag with a different prefix 38 | 6. Prints all other lines to console as they are being scanned 39 | 7. Only does it once, e.g., subsequent tag lines will end up with suffix 40 | 8. Yields None if no tag was found 41 | """ 42 | lines = template.split("\n") 43 | found_tag = False 44 | scan_prefix = None 45 | tag_pattern = re.compile(r"^\s*") # Precompile the regex pattern 46 | 47 | for line in lines: 48 | if not found_tag: 49 | if tag in line: 50 | found_tag = True 51 | # Match only the leading whitespace of the line with the tag 52 | match = tag_pattern.match(line) 53 | scan_prefix = match.group() if match else "" 54 | yield scan_prefix 55 | continue 56 | output_fd.write(line + "\n") 57 | else: 58 | if scan_prefix is not None: 59 | # Compute the current line's prefix 60 | current_line_prefix = tag_pattern.match(line).group() 61 | # Check if the current line's prefix is different from the tag's prefix 62 | if current_line_prefix == scan_prefix: 63 | continue 64 | scan_prefix = None 65 | output_fd.write(line + "\n") 66 | if not found_tag: 67 | yield None 68 | 69 | def is_git_repo(repo_path: str) -> bool: 70 | """ 71 | Check if the given path is a git repository. 72 | """ 73 | try: 74 | subprocess.check_output(["git", "-C", repo_path, "rev-parse", "--is-inside-work-tree"]) 75 | return True 76 | except subprocess.CalledProcessError: 77 | return False 78 | 79 | def get_api_url_from_git(repo_path: str) -> Optional[str]: 80 | if not is_git_repo(repo_path): 81 | raise ValueError( 82 | f"The path '{repo_path}' is not a git repository. Please provide the --github-url argument." 83 | ) 84 | """ 85 | Extract the GitHub API URL from the local git repository using git command. 86 | 87 | :param repo_path: Path to the local git repository. 88 | :return: GitHub API URL or None if not found. 89 | """ 90 | try: 91 | # Get the remote URL of the 'origin' remote repository 92 | git_url = ( 93 | subprocess.check_output( 94 | ["git", "-C", repo_path, "remote", "get-url", "origin"], 95 | stderr=subprocess.PIPE 96 | ) 97 | .decode() 98 | .strip() 99 | ) 100 | 101 | # Transform the git URL to the GitHub API URL 102 | if git_url.startswith("https://github.com/"): 103 | return git_url.replace( 104 | "https://github.com/", GITHUB_API_BASE_URL + "/", 1 105 | ).rstrip(".git") 106 | elif git_url.startswith("git@github.com:"): 107 | return git_url.replace( 108 | "git@github.com:", GITHUB_API_BASE_URL + "/", 1 109 | ).rstrip(".git") 110 | except Exception as e: 111 | print(f"Unexpected error: {e}", file=sys.stderr) 112 | return None 113 | 114 | 115 | def get_api_url(repo_url: Optional[str], local_repo: Optional[str]) -> str: 116 | """ 117 | Determine the GitHub API URL from either a repository URL or a local repository path. 118 | 119 | :param repo_url: GitHub repository URL. 120 | :param local_repo: Path to local Git repository. 121 | :return: GitHub API URL. 122 | :raises ValueError: If neither a repository URL nor a local repository path is provided. 123 | """ 124 | api_url = None 125 | if repo_url: 126 | api_url = repo_url.rstrip("/").replace("github.com", GITHUB_API_BASE_URL, 1) 127 | elif local_repo: 128 | api_url = get_api_url_from_git(local_repo) 129 | if api_url: 130 | if not api_url.startswith("https://"): 131 | api_url = f"https://{api_url}" 132 | return api_url 133 | else: 134 | raise ValueError( 135 | "Unable to determine the repository URL from the local Git repository." 136 | ) 137 | 138 | 139 | def github_request(request_url: str, method: str = 'GET', data: Optional[dict] = None) -> request.urlopen: 140 | """ 141 | Make a request to the GitHub API, supporting both GET and POST requests. 142 | This injects the GitHub API token environment variable into the request if present. 143 | 144 | :param request_url: URL to make the request to. 145 | :param method: HTTP method ('GET' or 'POST'). 146 | :param data: Data to be sent in the request body (for POST requests). 147 | :return: Response from the GitHub API. 148 | """ 149 | if data is not None: 150 | data = json.dumps(data).encode() 151 | req = request.Request(request_url, data=data, method=method) 152 | github_token = os.getenv("GITHUB_TOKEN") 153 | if github_token: 154 | auth_header = f"token {github_token}" 155 | req.add_header("Authorization", auth_header) 156 | req.add_header("Content-Type", "application/json") 157 | return request.urlopen(req) 158 | 159 | def fetch_contributors(api_url: str) -> List[str]: 160 | """ 161 | Fetch the list of contributors for a GitHub repository using GitHub's GraphQL API. 162 | If the GraphQL query fails, fallback to the REST API. 163 | 164 | :param api_url: GitHub API URL for the repository. 165 | :return: List of contributor usernames. 166 | """ 167 | graphql_url = "https://api.github.com/graphql" 168 | owner, repo = api_url.split('/')[-2:] 169 | query = """ 170 | query { 171 | repository(owner: "%s", name: "%s") { 172 | collaborators(first: 100) { 173 | edges { 174 | node { 175 | login 176 | } 177 | } 178 | } 179 | } 180 | } 181 | """ % (owner, repo) 182 | 183 | try: 184 | with github_request(graphql_url, 'POST', {'query': query}) as response: 185 | data = json.load(response) 186 | contributors = data['data']['repository']['collaborators']['edges'] 187 | return [contributor['node']['login'] for contributor in contributors] 188 | except (error.HTTPError, TypeError) as e: 189 | error_type = "HTTPError" if isinstance(e, error.HTTPError) else "KeyError" 190 | logging.error(f"{error_type} when querying {graphql_url}: {e}") 191 | logging.info("Attempting to list users via REST API as a fallback") 192 | return fetch_contributors_rest(api_url) 193 | 194 | 195 | def fetch_contributors_rest(api_url: str) -> List[str]: 196 | """ 197 | Fallback method to fetch the list of contributors for a GitHub repository using the REST API. 198 | 199 | :param api_url: GitHub API URL for the repository. 200 | :return: List of contributor usernames. 201 | """ 202 | url = f"{api_url}/contributors" 203 | try: 204 | with github_request(url) as response: 205 | contributors = json.load(response) 206 | logging.debug(f"{url} returned {json.dumps(contributors, indent=2)}") 207 | return [contributor["login"] for contributor in contributors] 208 | except error.HTTPError as e: 209 | logging.error(f"HTTP Error: {e.code} {e.reason}") 210 | logging.error( 211 | "For private repositories and to avoid throttling you must set the GITHUB_TOKEN. Alternatively, consider passing users explicitly via --github-users to avoid auth hassles." 212 | ) 213 | return [] 214 | 215 | 216 | def convert_key_to_age(key: str) -> Optional[str]: 217 | """ 218 | Convert an SSH key to an age key using ssh-to-age. 219 | 220 | :param key: The SSH key to convert. 221 | :return: The age key or None if conversion fails. 222 | """ 223 | try: 224 | result = subprocess.run( 225 | ["ssh-to-age"], input=key, stdout=subprocess.PIPE, text=True, check=True 226 | ) 227 | return result.stdout.strip() 228 | except Exception as e: 229 | print(f"Error running ssh-to-age: {e}", file=sys.stderr) 230 | return None 231 | 232 | 233 | def fetch_github_ssh_keys(contributors: List[str]) -> Dict[str, Dict[str, List[str]]]: 234 | """ 235 | Fetch and output the specified types of SSH keys for a list of GitHub users. 236 | Store each key type mapping to a list of keys. 237 | 238 | :param contributors: List of GitHub usernames. 239 | :return: A dictionary mapping usernames to dictionaries of key types and their keys. 240 | """ 241 | keys_by_user_and_type = {} 242 | for username in contributors: 243 | user_keys = keys_by_user_and_type.get(username, {}) 244 | try: 245 | with github_request(f"https://github.com/{username}.keys") as response: 246 | lines = response.read().decode().strip().splitlines() 247 | for line in lines: 248 | key_type, key = line.split(" ", 1) # Split on first space only 249 | if key_type not in user_keys: 250 | user_keys[key_type] = [] 251 | user_keys[key_type].append(key) 252 | keys_by_user_and_type[username] = user_keys 253 | except error.HTTPError as e: 254 | print( 255 | f"HTTP Error: {e.code} {e.reason} for user {username}", file=sys.stderr 256 | ) 257 | continue 258 | return keys_by_user_and_type 259 | 260 | 261 | def iterate_keys( 262 | keys: dict, 263 | accepted_key_types: Optional[Set[str]] = None, 264 | ): 265 | """ 266 | Print keys in useful formats 267 | 268 | :param key_types: The types of SSH keys to fetch (e.g., ['ssh-ed25519', 'ssh-rsa']) or None for all keys. 269 | :param convert_to_age: Whether to convert the keys to age keys. 270 | """ 271 | for username, user_keys in keys.items(): 272 | if accepted_key_types is not None: 273 | accepted_keys = set(user_keys.keys()).intersection(accepted_key_types) 274 | else: 275 | accepted_keys = user_keys.keys() 276 | if not accepted_keys: 277 | print( 278 | f"User {username} does not have any of the accepted key types: {','.join(list(accepted_key_types))}.", 279 | file=sys.stderr, 280 | ) 281 | for key_type in accepted_keys: 282 | key = user_keys[key_type] 283 | yield {"username": username, "key_type": key_type, "key": key} 284 | 285 | def ssh_keyscan(hosts: List[str], parsed_keys: Dict[str, Dict[str, List[str]]] = None) -> Dict[str, Dict[str, List[str]]]: 286 | """ 287 | Perform an SSH key scan for a list of hosts and parse the known hosts content. 288 | 289 | :param hosts: A list of hostnames or IP addresses to scan. 290 | :param parsed_keys: An optional dictionary to which the scan results will be added. 291 | If not provided, a new dictionary will be created. 292 | :return: A dictionary mapping each host to a dictionary of key types and their keys. 293 | Each key type maps to a list of keys to accommodate multiple keys of the same type. 294 | """ 295 | if parsed_keys is None: 296 | parsed_keys = {} 297 | 298 | def ssh_keyscan_inner(host: str) -> str: 299 | """ 300 | Run the ssh-keyscan command for a single host and return its output. 301 | 302 | :param host: The hostname or IP address to scan. 303 | :return: The stdout from the ssh-keyscan command. 304 | :raises Exception: If ssh-keyscan fails. 305 | """ 306 | try: 307 | result = subprocess.run( 308 | ["ssh-keyscan", host], 309 | check=True, 310 | stdout=subprocess.PIPE, 311 | text=True 312 | ) 313 | return result.stdout 314 | except subprocess.CalledProcessError as e: 315 | raise Exception(f"ssh-keyscan failed with exit code {e.returncode}: {e.stderr}") 316 | 317 | def parse_known_hosts_content(known_hosts: str, parsed_keys: Dict[str, Dict[str, List[str]]]): 318 | """ 319 | Parse the content of known hosts and update the parsed_keys dictionary. 320 | 321 | :param known_hosts: The stdout from the ssh-keyscan command. 322 | :param parsed_keys: The dictionary to which the scan results will be added. 323 | """ 324 | for line in known_hosts.splitlines(): 325 | if line.startswith("#") or line.strip() == "": 326 | continue 327 | 328 | parts = line.strip().split() 329 | if len(parts) < 3: 330 | continue 331 | 332 | host, key_type, key = parts[0], parts[1], parts[2] 333 | if host not in parsed_keys: 334 | parsed_keys[host] = {} 335 | if key_type not in parsed_keys[host]: 336 | parsed_keys[host][key_type] = [] 337 | parsed_keys[host][key_type].append(key) 338 | 339 | for host in hosts: 340 | known_hosts_log = ssh_keyscan_inner(host) 341 | parse_known_hosts_content(known_hosts_log, parsed_keys) 342 | 343 | return parsed_keys 344 | 345 | def is_tool_available(name): 346 | """Check if a tool is available on the system.""" 347 | try: 348 | subprocess.run( 349 | [name, "--version"], 350 | stdout=subprocess.PIPE, 351 | stderr=subprocess.PIPE, 352 | check=True, 353 | ) 354 | return True 355 | except (OSError, subprocess.CalledProcessError): 356 | return False 357 | 358 | 359 | def comma_separated_list(string: str) -> Set[str]: 360 | """ 361 | Converts a comma-separated string into a set of strings. 362 | 363 | :param string: A string containing comma-separated values. 364 | :return: A set containing the individual values as strings. 365 | """ 366 | return set(string.split(",")) 367 | 368 | 369 | def print_keys(template: str, user_keys: Dict[str, Dict[str, List[str]]], 370 | accepted_key_types: Set[str], output_format: str, 371 | output_fd: TextIO) -> None: 372 | """ 373 | Processes a template and prints SSH keys in a specified format. 374 | 375 | :param template: A string template for processing. 376 | :param user_keys: A dictionary mapping usernames to dictionaries of key types and their keys. 377 | Each key type maps to a list of keys. 378 | :param accepted_key_types: A set of accepted key types. 379 | :param output_format: The format in which to output the keys. 380 | :param output_fd: The file descriptor to write the output to. 381 | """ 382 | # Assuming process_template is a function you have defined elsewhere 383 | for line_prefix in process_template(template, GITHUB_TO_SOPS_TAG, output_fd): 384 | if line_prefix is None: 385 | line_prefix = "" 386 | print(f"{line_prefix}# {GENERATED_MSG}", file=output_fd) 387 | 388 | # Sort the users by their username 389 | sorted_users = sorted(user_keys.keys(), key=lambda username: username.lower()) 390 | 391 | for username in sorted_users: 392 | user_key_types = user_keys[username] 393 | for key_type in user_key_types: 394 | if accepted_key_types is not None and key_type not in accepted_key_types: 395 | continue 396 | for key in user_key_types[key_type]: 397 | if output_format in ["ssh-to-age", "sops"]: 398 | # Assuming convert_key_to_age is a function you have defined elsewhere 399 | key = convert_key_to_age(f"{key_type} {key}") 400 | if not key: 401 | print( 402 | f"Skipped converting {key_type} key for user {username} to age key with ssh-to-age", 403 | file=sys.stderr, 404 | ) 405 | continue 406 | if output_format == "sops": 407 | print(f"{line_prefix}- {key} # {username}", file=output_fd) 408 | else: 409 | print(f"{key}", file=output_fd) 410 | else: 411 | print(f"{key_type} {key} {username}", file=output_fd) 412 | 413 | 414 | def refresh_secrets(args): 415 | """ 416 | Find all .sops.yaml files in the repo that are managed by git and run `import-keys --inplace-edit .sops.yaml` on them. 417 | """ 418 | import subprocess 419 | import os 420 | import logging 421 | 422 | # Configure logging to output to stderr 423 | logging.basicConfig(level=logging.INFO, format='%(message)s', stream=sys.stderr) 424 | 425 | def find_sops_yaml_files(): 426 | """ 427 | Find all .sops.yaml files in the repo that are managed by git. 428 | """ 429 | logging.info("Finding .sops.yaml files in the repo managed by git.") 430 | result = subprocess.run( 431 | ["git", "ls-files", "*.sops.yaml"], 432 | stdout=subprocess.PIPE, 433 | text=True, 434 | check=True 435 | ) 436 | return result.stdout.splitlines() 437 | 438 | def find_enc_yaml_files(): 439 | """ 440 | Find all *.enc.yaml files in the repo that are managed by git. 441 | """ 442 | logging.info("Finding *.enc.yaml files in the repo managed by git.") 443 | result = subprocess.run( 444 | ["git", "ls-files", "*.enc.yaml"], 445 | stdout=subprocess.PIPE, 446 | text=True, 447 | check=True 448 | ) 449 | return result.stdout.splitlines() 450 | 451 | def file_contains_sops(file_path): 452 | """ 453 | Check if the file contains 'sops:' in its content. 454 | """ 455 | with open(file_path, 'r') as file: 456 | return 'sops:' in file.read() 457 | 458 | sops_yaml_files = find_sops_yaml_files() 459 | logging.info(f"Found {len(sops_yaml_files)} .sops.yaml files.") 460 | for file in sops_yaml_files: 461 | logging.info(f"Running import-keys --inplace-edit on {file}.") 462 | subprocess.run( 463 | [sys.argv[0], "import-keys", "--inplace-edit", file], 464 | check=True 465 | ) 466 | 467 | enc_yaml_files = find_enc_yaml_files() 468 | logging.info(f"Found {len(enc_yaml_files)} *.enc.yaml files.") 469 | for file in enc_yaml_files: 470 | if file_contains_sops(file): 471 | logging.info(f"Running sops updatekeys -y on {file}.") 472 | subprocess.run( 473 | ["sops", "updatekeys", "-y", file], 474 | check=True 475 | ) 476 | 477 | def generate_keys(args): 478 | """ 479 | Main func 480 | """ 481 | if args.inplace_edit: 482 | args.format = "sops" 483 | input_template = open(args.inplace_edit, "r").read() 484 | output_fd = open(args.inplace_edit + ".tmp", "w") 485 | if args.key_types is None: 486 | args.key_types = set(["ssh-ed25519"]) 487 | 488 | api_url = get_api_url(args.github_url, args.local_github_checkout) 489 | if args.github_users: 490 | contributors = args.github_users 491 | else: 492 | contributors = fetch_contributors(api_url) 493 | 494 | keys = fetch_github_ssh_keys(contributors) 495 | 496 | if args.ssh_hosts: 497 | keys = ssh_keyscan(args.ssh_hosts, keys) 498 | 499 | print_keys( 500 | template=input_template.strip() if args.inplace_edit and args.format == "sops" else SOPS_TEMPLATE if args.format == "sops" else "", 501 | user_keys=keys, 502 | accepted_key_types=args.key_types, 503 | output_format=args.format, 504 | output_fd=output_fd if args.inplace_edit else sys.stdout, 505 | ) 506 | if args.inplace_edit: 507 | output_fd.close() 508 | os.rename(args.inplace_edit + ".tmp", args.inplace_edit) 509 | 510 | def run_sops(): 511 | """Run sops with SOPS_AGE_KEY set from ~/.ssh/id_ed25519""" 512 | try: 513 | # Read SSH private key 514 | with open(os.path.expanduser("~/.ssh/id_ed25519"), "r") as key_file: 515 | # Convert to AGE key 516 | result = subprocess.run( 517 | ["ssh-to-age", "-private-key"], 518 | stdin=key_file, 519 | stdout=subprocess.PIPE, 520 | stderr=subprocess.PIPE, 521 | text=True, 522 | check=True 523 | ) 524 | age_key = result.stdout.strip() 525 | 526 | # Set environment and exec sops 527 | os.environ["SOPS_AGE_KEY"] = age_key 528 | 529 | # Pass through all arguments after 'sops' to the sops command 530 | os.execvp("sops", ["sops"] + sys.argv[2:]) 531 | 532 | except Exception as e: 533 | print(f"Error: {e}", file=sys.stderr) 534 | sys.exit(1) 535 | 536 | def get_version(): 537 | try: 538 | from importlib.metadata import version 539 | return version("github_to_sops") 540 | except Exception: 541 | return "unknown" 542 | 543 | def get_goos(system): 544 | """ 545 | Returns the GOOS value based on the system. 546 | 547 | :param system: The operating system. 548 | :return: The GOOS value. 549 | """ 550 | goos_map = { 551 | "Linux": "linux", 552 | "Darwin": "darwin" 553 | } 554 | return goos_map.get(system) 555 | 556 | def get_goarch(machine): 557 | """ 558 | Returns the GOARCH value based on the machine. 559 | 560 | :param machine: The machine architecture. 561 | :return: The GOARCH value. 562 | """ 563 | goarch_map = { 564 | "x86_64": "amd64", 565 | "aarch64": "arm64", 566 | "arm64": "arm64" 567 | } 568 | return goarch_map.get(machine) 569 | 570 | def get_sops_download_url(system, machine, version="v3.9.0"): 571 | """ 572 | Returns the download URL for the sops binary based on the system, machine, and version. 573 | 574 | :param system: The operating system. 575 | :param machine: The machine architecture. 576 | :param version: The version of the sops binary. 577 | :return: The download URL for the sops binary. 578 | """ 579 | goos = get_goos(system) 580 | goarch = get_goarch(machine) 581 | if goos and goarch: 582 | base_url = f"https://github.com/getsops/sops/releases/download/{version}/sops-{version}" 583 | return f"{base_url}.{goos}.{goarch}" 584 | return None 585 | return None 586 | 587 | def install_binaries(args): 588 | import os 589 | import platform 590 | import subprocess 591 | import tempfile 592 | import urllib.request 593 | import shutil 594 | 595 | def is_binary_installed(name): 596 | """Check if a binary is already installed and executable""" 597 | return shutil.which(name) is not None 598 | 599 | def run_docker_command(goos, goarch): 600 | if is_binary_installed("ssh-to-age"): 601 | print("ssh-to-age is already installed, skipping installation") 602 | return 603 | 604 | if not is_binary_installed("docker"): 605 | print("Docker is required to build ssh-to-age but is not available", file=sys.stderr) 606 | sys.exit(1) 607 | 608 | temp_dir = tempfile.gettempdir() 609 | temp_output_path = os.path.join(temp_dir, "output") 610 | os.makedirs(temp_output_path, exist_ok=True) 611 | docker_command = [ 612 | "docker", "run", "--rm", 613 | "-e", f"GOOS={goos}", 614 | "-e", f"GOARCH={goarch}", 615 | "-v", f"{temp_output_path}:/output", 616 | "golang:latest", 617 | "sh", "-c", 618 | 'git clone --branch 1.1.8 https://github.com/Mic92/ssh-to-age.git /src && cd /src/cmd/ssh-to-age && go build && find /src -type f -name ssh-to-age -exec cp {} /output/ \\;' 619 | ] 620 | print(f"Executing: {' '.join(f'{arg}' for arg in docker_command)}") 621 | subprocess.run(docker_command, check=True) 622 | temp_binary_path = os.path.join(temp_output_path, "ssh-to-age") 623 | try: 624 | print(f"Executing: sudo mv {temp_binary_path} /usr/local/bin/ssh-to-age") 625 | subprocess.run(["sudo", "mv", temp_binary_path, "/usr/local/bin/ssh-to-age"], check=True) 626 | print("ssh-to-age binary installed successfully to /usr/local/bin/ssh-to-age") 627 | except subprocess.CalledProcessError as e: 628 | print(f"Failed to move ssh-to-age binary to /usr/local/bin/ssh-to-age: {e}", file=sys.stderr) 629 | sys.exit(1) 630 | 631 | def download_and_install_sops(system, machine): 632 | if is_binary_installed("sops"): 633 | print("sops is already installed, skipping installation") 634 | return 635 | 636 | download_url = get_sops_download_url(system, machine) 637 | if download_url is None: 638 | print("Not supported on your platform", file=sys.stderr) 639 | sys.exit(1) 640 | 641 | temp_dir = tempfile.gettempdir() 642 | binary_name = "sops" 643 | temp_binary_path = os.path.join(temp_dir, binary_name) 644 | 645 | print(f"Downloading sops binary from {download_url} to {temp_binary_path}") 646 | with urllib.request.urlopen(download_url) as response, open(temp_binary_path, 'wb') as out_file: 647 | shutil.copyfileobj(response, out_file) 648 | print("Download completed") 649 | 650 | os.chmod(temp_binary_path, 0o755) 651 | try: 652 | print(f"Executing: sudo mv {temp_binary_path} /usr/local/bin/sops") 653 | subprocess.run(["sudo", "mv", temp_binary_path, "/usr/local/bin/sops"], check=True) 654 | print("sops binary installed successfully to /usr/local/bin/sops") 655 | except subprocess.CalledProcessError as e: 656 | print(f"Failed to move sops binary to /usr/local/bin/sops: {e}", file=sys.stderr) 657 | sys.exit(1) 658 | 659 | system = platform.system() 660 | machine = platform.machine() 661 | goos = get_goos(system) 662 | goarch = get_goarch(machine) 663 | 664 | if goos is None or goarch is None: 665 | print("Not supported on your platform", file=sys.stderr) 666 | sys.exit(1) 667 | 668 | run_docker_command(goos, goarch) 669 | download_and_install_sops(system, machine) 670 | 671 | def main(): 672 | # Handle sops subcommand before argparse, otherwise argparse will make it hard to pass arguments to sops 673 | if len(sys.argv) > 1 and sys.argv[1] == "sops": 674 | run_sops() 675 | 676 | parser = argparse.ArgumentParser( 677 | description="Manage GitHub SSH keys and generate SOPS-compatible SSH key files.", 678 | add_help=False 679 | ) 680 | parser.add_argument( 681 | "--version", 682 | action="version", 683 | version=f"%(prog)s {get_version()}" 684 | ) 685 | parser.add_argument( 686 | "-h", "--help", 687 | action="help", 688 | help="Show this help message and exit." 689 | ) 690 | subparsers = parser.add_subparsers(dest="command") 691 | 692 | # Add sops subcommand documentation 693 | sops_parser = subparsers.add_parser( 694 | "sops", 695 | help="Run sops with SOPS_AGE_KEY set from ~/.ssh/id_ed25519", 696 | description="""Run sops with SOPS_AGE_KEY environment variable set by converting 697 | your SSH private key (~/.ssh/id_ed25519) to an AGE key using ssh-to-age. 698 | This allows seamless SOPS operations using your existing SSH key. 699 | Note: Requires ssh-to-age to be installed.""", 700 | add_help=False 701 | ) 702 | sops_parser.add_argument( 703 | "-h", "--help", 704 | action="help", 705 | help="Show this help message and exit." 706 | ) 707 | 708 | install_binaries_parser = subparsers.add_parser( 709 | "install-binaries", 710 | help="Install ssh-to-age, sops binaries for supported platforms (Linux and Mac)." 711 | ) 712 | install_binaries_parser.add_argument( 713 | "-v", 714 | "--verbose", 715 | help="Turn on debug logging to see HTTP requests and other internal Python stuff.", 716 | action="store_true", 717 | ) 718 | 719 | refresh_secrets_parser = subparsers.add_parser( 720 | "refresh-secrets", 721 | help="Find all .sops.yaml files in the repo that are managed by git and run `import-keys --inplace-edit .sops.yaml` on them." 722 | ) 723 | refresh_secrets_parser.add_argument( 724 | "-v", 725 | "--verbose", 726 | help="Turn on debug logging to see HTTP requests and other internal Python stuff.", 727 | action="store_true", 728 | ) 729 | 730 | import_keys_parser = subparsers.add_parser( 731 | "import-keys", 732 | help="Import SSH keys of GitHub repository contributors or specified github users and output that info into a useful format like sops or ssh authorized_keys", 733 | epilog=f"""Example invocations: 734 | `{sys.argv[0]} import-keys --github-url https://github.com/tarasglek/chatcraft.org --key-types ssh-ed25519 --format sops` 735 | `{sys.argv[0]} import-keys --github-url https://github.com/tarasglek/chatcraft.org --format authorized_keys` 736 | `{sys.argv[0]} import-keys --local-github-checkout . --format sops --ssh-hosts 192.168.1.1,192.168.1.2 --key-types ssh-ed25519` 737 | """, 738 | ) 739 | import_keys_parser.add_argument("--github-url", help="GitHub repository URL.") 740 | import_keys_parser.add_argument("--local-github-checkout", default=".", help="Path to local Git repository.") 741 | import_keys_parser.add_argument( 742 | "--ssh-hosts", 743 | type=comma_separated_list, 744 | help="Comma-separated list of ssh servers to fetch public keys from." 745 | ) 746 | import_keys_parser.add_argument( 747 | "--github-users", 748 | type=comma_separated_list, 749 | help="Comma-separated list of GitHub usernames to fetch keys for.", 750 | ) 751 | import_keys_parser.add_argument( 752 | "--key-types", 753 | type=comma_separated_list, 754 | default=None, 755 | help="Comma-separated types of SSH keys to fetch (e.g., ssh-ed25519,ssh-rsa). Pass no value for all types.", 756 | ) 757 | # Supported conversions with validation 758 | supported_conversions = ["authorized_keys", "ssh-to-age", "sops"] 759 | import_keys_parser.add_argument( 760 | "--format", 761 | default="sops", 762 | type=str, 763 | choices=supported_conversions, 764 | help=f"Output/convert keys using the specified format. Supported formats: " 765 | f"{', '.join(supported_conversions)}. For example, use '--format " 766 | f"ssh-to-age' to convert SSH keys to age keys.", 767 | ) 768 | import_keys_parser.add_argument( 769 | "--inplace-edit", 770 | help="Edit SOPS file in-place. This takes a .sops.yaml file as input and replaces it. This sets --format to sops", 771 | ) 772 | import_keys_parser.add_argument( 773 | "-v", 774 | "--verbose", 775 | help="Turn on debug logging to see HTTP requests and other internal Python stuff.", 776 | action="store_true", 777 | ) 778 | 779 | args = parser.parse_args() 780 | if args.command == "install-binaries": 781 | install_binaries(args) 782 | elif args.command == "refresh-secrets": 783 | refresh_secrets(args) 784 | elif args.command == "import-keys": 785 | generate_keys(args) 786 | else: 787 | parser.print_help() 788 | 789 | if __name__ == "__main__": 790 | main() 791 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "github-to-sops" 3 | version = "1.4.1" 4 | description = "Standalone (no-dependencies beyond Python) script fetches SSH keys of GitHub repository contributors and generates SOPS-compatible SSH key files." 5 | readme = "README.md" 6 | requires-python = ">=3.8" 7 | authors = [{name = "Taras Glek"}] 8 | license = {text = "MIT"} 9 | classifiers = [ 10 | "License :: OSI Approved :: MIT License" 11 | ] 12 | dependencies = [ 13 | 14 | ] 15 | 16 | [build-system] 17 | requires = ["setuptools", "wheel"] 18 | build-backend = "setuptools.build_meta" 19 | 20 | [tool.setuptools] 21 | packages = ["github_to_sops"] 22 | 23 | [project.scripts] 24 | github-to-sops = "github_to_sops.__init__:main" 25 | 26 | [project.urls] 27 | Homepage = "https://github.com/tarasglek/github-to-sops" 28 | Changelog = "https://github.com/tarasglek/github-to-sops/releases" 29 | Issues = "https://github.com/tarasglek/github-to-sops/issues" 30 | CI = "https://github.com/tarasglek/github-to-sops/actions" 31 | --------------------------------------------------------------------------------