├── .ansible-lint ├── .bandit.yml ├── .coveragerc ├── .flake8 ├── .github ├── CODEOWNERS ├── dependabot.yml ├── labels.yml ├── lineage.yml └── workflows │ ├── build.yml │ ├── codeql-analysis.yml │ ├── dependency-review.yml │ └── sync-labels.yml ├── .gitignore ├── .isort.cfg ├── .mdl_config.yaml ├── .pre-commit-config.yaml ├── .prettierignore ├── .yamllint ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── bump-version ├── pytest.ini ├── requirements-dev.txt ├── requirements-test.txt ├── requirements.txt ├── run ├── setup-env ├── setup.py ├── src └── trustymail │ ├── __init__.py │ ├── __main__.py │ ├── _version.py │ ├── cli.py │ ├── domain.py │ └── trustymail.py ├── tag.sh └── tests ├── conftest.py └── test_trustymail.py /.ansible-lint: -------------------------------------------------------------------------------- 1 | --- 2 | # See https://ansible-lint.readthedocs.io/configuring/ for a list of 3 | # the configuration elements that can exist in this file. 4 | enable_list: 5 | # Useful checks that one must opt-into. See here for more details: 6 | # https://ansible-lint.readthedocs.io/rules/ 7 | - fcqn-builtins 8 | - no-log-password 9 | - no-same-owner 10 | exclude_paths: 11 | # This exclusion is implicit, unless exclude_paths is defined 12 | - .cache 13 | # Seems wise to ignore this too 14 | - .github 15 | kinds: 16 | # This will force our systemd specific molecule configurations to be treated 17 | # as plain yaml files by ansible-lint. This mirrors the default kind 18 | # configuration in ansible-lint for molecule configurations: 19 | # yaml: "**/molecule/*/{base,molecule}.{yaml,yml}" 20 | - yaml: "**/molecule/*/molecule-{no,with}-systemd.yml" 21 | use_default_rules: true 22 | -------------------------------------------------------------------------------- /.bandit.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configuration file for the Bandit python security scanner 3 | # https://bandit.readthedocs.io/en/latest/config.html 4 | # This config is applied to bandit when scanning the "tests" tree 5 | 6 | # Tests are first included by `tests`, and then excluded by `skips`. 7 | # If `tests` is empty, all tests are considered included. 8 | 9 | tests: 10 | # - B101 11 | # - B102 12 | 13 | skips: 14 | - B101 # skip "assert used" check since assertions are required in pytests 15 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | # This is the configuration for code coverage checks 2 | # https://coverage.readthedocs.io/en/latest/config.html 3 | 4 | [run] 5 | source = src/trustymail 6 | omit = 7 | branch = true 8 | 9 | [report] 10 | exclude_lines = 11 | if __name__ == "__main__": 12 | show_missing = true 13 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 80 3 | # Select (turn on) 4 | # * Complexity violations reported by mccabe (C) - 5 | # http://flake8.pycqa.org/en/latest/user/error-codes.html#error-violation-codes 6 | # * Documentation conventions compliance reported by pydocstyle (D) - 7 | # http://www.pydocstyle.org/en/stable/error_codes.html 8 | # * Default errors and warnings reported by pycodestyle (E and W) - 9 | # https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes 10 | # * Default errors reported by pyflakes (F) - 11 | # http://flake8.pycqa.org/en/latest/glossary.html#term-pyflakes 12 | # * Default warnings reported by flake8-bugbear (B) - 13 | # https://github.com/PyCQA/flake8-bugbear#list-of-warnings 14 | # * The B950 flake8-bugbear opinionated warning - 15 | # https://github.com/PyCQA/flake8-bugbear#opinionated-warnings 16 | select = C,D,E,F,W,B,B950 17 | # Ignore flake8's default warning about maximum line length, which has 18 | # a hard stop at the configured value. Instead we use 19 | # flake8-bugbear's B950, which allows up to 10% overage. 20 | # 21 | # Also ignore flake8's warning about line breaks before binary 22 | # operators. It no longer agrees with PEP8. See, for example, here: 23 | # https://github.com/ambv/black/issues/21. Guido agrees here: 24 | # https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b. 25 | ignore = E501,W503 26 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Each line is a file pattern followed by one or more owners. 2 | 3 | # These owners will be the default owners for everything in the 4 | # repo. Unless a later match takes precedence, these owners will be 5 | # requested for review when someone opens a pull request. 6 | * @dav3r @felddy @jsf9k @mcdonnnj 7 | 8 | # These folks own any files in the .github directory at the root of 9 | # the repository and any of its subdirectories. 10 | /.github/ @dav3r @felddy @jsf9k @mcdonnnj 11 | 12 | # These folks own all linting configuration files. 13 | /.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj 14 | /.bandit.yml @dav3r @felddy @jsf9k @mcdonnnj 15 | /.flake8 @dav3r @felddy @jsf9k @mcdonnnj 16 | /.isort.cfg @dav3r @felddy @jsf9k @mcdonnnj 17 | /.mdl_config.yaml @dav3r @felddy @jsf9k @mcdonnnj 18 | /.pre-commit-config.yaml @dav3r @felddy @jsf9k @mcdonnnj 19 | /.prettierignore @dav3r @felddy @jsf9k @mcdonnnj 20 | /.yamllint @dav3r @felddy @jsf9k @mcdonnnj 21 | /requirements.txt @dav3r @felddy @jsf9k @mcdonnnj 22 | /requirements-dev.txt @dav3r @felddy @jsf9k @mcdonnnj 23 | /requirements-test.txt @dav3r @felddy @jsf9k @mcdonnnj 24 | /setup-env @dav3r @felddy @jsf9k @mcdonnnj 25 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Any ignore directives should be uncommented in downstream projects to disable 4 | # Dependabot updates for the given dependency. Downstream projects will get 5 | # these updates when the pull request(s) in the appropriate skeleton are merged 6 | # and Lineage processes these changes. 7 | 8 | updates: 9 | - directory: / 10 | ignore: 11 | # Managed by cisagov/skeleton-generic 12 | - dependency-name: actions/cache 13 | - dependency-name: actions/checkout 14 | - dependency-name: actions/dependency-review-action 15 | - dependency-name: actions/setup-go 16 | - dependency-name: actions/setup-python 17 | - dependency-name: cisagov/action-job-preamble 18 | - dependency-name: cisagov/setup-env-github-action 19 | - dependency-name: crazy-max/ghaction-github-labeler 20 | - dependency-name: github/codeql-action 21 | - dependency-name: hashicorp/setup-packer 22 | - dependency-name: hashicorp/setup-terraform 23 | - dependency-name: mxschmitt/action-tmate 24 | # Managed by cisagov/skeleton-python-library 25 | - dependency-name: actions/download-artifact 26 | - dependency-name: actions/upload-artifact 27 | package-ecosystem: github-actions 28 | schedule: 29 | interval: weekly 30 | 31 | - directory: / 32 | package-ecosystem: pip 33 | schedule: 34 | interval: weekly 35 | 36 | - directory: / 37 | package-ecosystem: terraform 38 | schedule: 39 | interval: weekly 40 | version: 2 41 | -------------------------------------------------------------------------------- /.github/labels.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Rather than breaking up descriptions into multiline strings we disable that 3 | # specific rule in yamllint for this file. 4 | # yamllint disable rule:line-length 5 | - color: eb6420 6 | description: This issue or pull request is awaiting the outcome of another issue or pull request 7 | name: blocked 8 | - color: "000000" 9 | description: This issue or pull request involves changes to existing functionality 10 | name: breaking change 11 | - color: d73a4a 12 | description: This issue or pull request addresses broken functionality 13 | name: bug 14 | - color: 07648d 15 | description: This issue will be advertised on code.gov's Open Tasks page (https://code.gov/open-tasks) 16 | name: code.gov 17 | - color: 0366d6 18 | description: Pull requests that update a dependency file 19 | name: dependencies 20 | - color: 5319e7 21 | description: This issue or pull request improves or adds to documentation 22 | name: documentation 23 | - color: cfd3d7 24 | description: This issue or pull request already exists or is covered in another issue or pull request 25 | name: duplicate 26 | - color: b005bc 27 | description: A high-level objective issue encompassing multiple issues instead of a specific unit of work 28 | name: epic 29 | - color: "000000" 30 | description: Pull requests that update GitHub Actions code 31 | name: github-actions 32 | - color: 0e8a16 33 | description: This issue or pull request is well-defined and good for newcomers 34 | name: good first issue 35 | - color: ff7518 36 | description: Pull request that should count toward Hacktoberfest participation 37 | name: hacktoberfest-accepted 38 | - color: a2eeef 39 | description: This issue or pull request will add or improve functionality, maintainability, or ease of use 40 | name: improvement 41 | - color: fef2c0 42 | description: This issue or pull request is not applicable, incorrect, or obsolete 43 | name: invalid 44 | - color: ce099a 45 | description: This pull request is ready to merge during the next Lineage Kraken release 46 | name: kraken 🐙 47 | - color: a4fc5d 48 | description: This issue or pull request requires further information 49 | name: need info 50 | - color: fcdb45 51 | description: This pull request is awaiting an action or decision to move forward 52 | name: on hold 53 | - color: 3772a4 54 | description: Pull requests that update Python code 55 | name: python 56 | - color: ef476c 57 | description: This issue is a request for information or needs discussion 58 | name: question 59 | - color: d73a4a 60 | description: This issue or pull request addresses a security issue 61 | name: security 62 | - color: 00008b 63 | description: This issue or pull request adds or otherwise modifies test code 64 | name: test 65 | - color: 1d76db 66 | description: This issue or pull request pulls in upstream updates 67 | name: upstream update 68 | - color: d4c5f9 69 | description: This issue or pull request increments the version number 70 | name: version bump 71 | - color: ffffff 72 | description: This issue will not be incorporated 73 | name: wontfix 74 | -------------------------------------------------------------------------------- /.github/lineage.yml: -------------------------------------------------------------------------------- 1 | --- 2 | lineage: 3 | skeleton: 4 | remote-url: https://github.com/cisagov/skeleton-python-library.git 5 | version: "1" 6 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: build 3 | 4 | on: # yamllint disable-line rule:truthy 5 | merge_group: 6 | types: 7 | - checks_requested 8 | pull_request: 9 | push: 10 | repository_dispatch: 11 | types: 12 | - apb 13 | 14 | # Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, 15 | # nounset, errexit, and pipefail. The `-x` will print all commands as they are 16 | # run. Please see the GitHub Actions documentation for more information: 17 | # https://docs.github.com/en/actions/using-jobs/setting-default-values-for-jobs 18 | defaults: 19 | run: 20 | shell: bash -Eueo pipefail -x {0} 21 | 22 | env: 23 | PIP_CACHE_DIR: ~/.cache/pip 24 | PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit 25 | RUN_TMATE: ${{ secrets.RUN_TMATE }} 26 | TERRAFORM_DOCS_REPO_BRANCH_NAME: improvement/support_atx_closed_markdown_headers 27 | TERRAFORM_DOCS_REPO_DEPTH: 1 28 | TERRAFORM_DOCS_REPO_URL: https://github.com/mcdonnnj/terraform-docs.git 29 | 30 | jobs: 31 | diagnostics: 32 | name: Run diagnostics 33 | # This job does not need any permissions 34 | permissions: {} 35 | runs-on: ubuntu-latest 36 | steps: 37 | # Note that a duplicate of this step must be added at the top of 38 | # each job. 39 | - name: Apply standard cisagov job preamble 40 | uses: cisagov/action-job-preamble@v1 41 | with: 42 | check_github_status: "true" 43 | # This functionality is poorly implemented and has been 44 | # causing problems due to the MITM implementation hogging or 45 | # leaking memory. As a result we disable it by default. If 46 | # you want to temporarily enable it, simply set 47 | # monitor_permissions equal to "true". 48 | # 49 | # TODO: Re-enable this functionality when practical. See 50 | # cisagov/skeleton-generic#207 for more details. 51 | monitor_permissions: "false" 52 | output_workflow_context: "true" 53 | # Use a variable to specify the permissions monitoring 54 | # configuration. By default this will yield the 55 | # configuration stored in the cisagov organization-level 56 | # variable, but if you want to use a different configuration 57 | # then simply: 58 | # 1. Create a repository-level variable with the name 59 | # ACTIONS_PERMISSIONS_CONFIG. 60 | # 2. Set this new variable's value to the configuration you 61 | # want to use for this repository. 62 | # 63 | # Note in particular that changing the permissions 64 | # monitoring configuration *does not* require you to modify 65 | # this workflow. 66 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 67 | lint: 68 | needs: 69 | - diagnostics 70 | permissions: 71 | # actions/checkout needs this to fetch code 72 | contents: read 73 | runs-on: ubuntu-latest 74 | steps: 75 | - name: Apply standard cisagov job preamble 76 | uses: cisagov/action-job-preamble@v1 77 | with: 78 | # This functionality is poorly implemented and has been 79 | # causing problems due to the MITM implementation hogging or 80 | # leaking memory. As a result we disable it by default. If 81 | # you want to temporarily enable it, simply set 82 | # monitor_permissions equal to "true". 83 | # 84 | # TODO: Re-enable this functionality when practical. See 85 | # cisagov/skeleton-generic#207 for more details. 86 | monitor_permissions: "false" 87 | # Use a variable to specify the permissions monitoring 88 | # configuration. By default this will yield the 89 | # configuration stored in the cisagov organization-level 90 | # variable, but if you want to use a different configuration 91 | # then simply: 92 | # 1. Create a repository-level variable with the name 93 | # ACTIONS_PERMISSIONS_CONFIG. 94 | # 2. Set this new variable's value to the configuration you 95 | # want to use for this repository. 96 | # 97 | # Note in particular that changing the permissions 98 | # monitoring configuration *does not* require you to modify 99 | # this workflow. 100 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 101 | - id: setup-env 102 | uses: cisagov/setup-env-github-action@develop 103 | - uses: actions/checkout@v4 104 | - id: setup-python 105 | uses: actions/setup-python@v5 106 | with: 107 | python-version: ${{ steps.setup-env.outputs.python-version }} 108 | # We need the Go version and Go cache location for the actions/cache step, 109 | # so the Go installation must happen before that. 110 | - id: setup-go 111 | uses: actions/setup-go@v5 112 | with: 113 | # There is no expectation for actual Go code so we disable caching as 114 | # it relies on the existence of a go.sum file. 115 | cache: false 116 | go-version: ${{ steps.setup-env.outputs.go-version }} 117 | - id: go-cache 118 | name: Lookup Go cache directory 119 | run: | 120 | echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT 121 | - uses: actions/cache@v4 122 | env: 123 | BASE_CACHE_KEY: ${{ github.job }}-${{ runner.os }}-\ 124 | py${{ steps.setup-python.outputs.python-version }}-\ 125 | go${{ steps.setup-go.outputs.go-version }}-\ 126 | packer${{ steps.setup-env.outputs.packer-version }}-\ 127 | tf${{ steps.setup-env.outputs.terraform-version }}- 128 | with: 129 | # We do not use '**/setup.py' in the cache key so only the 'setup.py' 130 | # file in the root of the repository is used. This is in case a Python 131 | # package were to have a 'setup.py' as part of its internal codebase. 132 | key: ${{ env.BASE_CACHE_KEY }}\ 133 | ${{ hashFiles('**/requirements-test.txt') }}-\ 134 | ${{ hashFiles('**/requirements.txt') }}-\ 135 | ${{ hashFiles('**/.pre-commit-config.yaml') }}-\ 136 | ${{ hashFiles('setup.py') }} 137 | # Note that the .terraform directory IS NOT included in the 138 | # cache because if we were caching, then we would need to use 139 | # the `-upgrade=true` option. This option blindly pulls down the 140 | # latest modules and providers instead of checking to see if an 141 | # update is required. That behavior defeats the benefits of caching. 142 | # so there is no point in doing it for the .terraform directory. 143 | path: | 144 | ${{ env.PIP_CACHE_DIR }} 145 | ${{ env.PRE_COMMIT_CACHE_DIR }} 146 | ${{ steps.go-cache.outputs.dir }} 147 | restore-keys: | 148 | ${{ env.BASE_CACHE_KEY }} 149 | - uses: hashicorp/setup-packer@v3 150 | with: 151 | version: ${{ steps.setup-env.outputs.packer-version }} 152 | - uses: hashicorp/setup-terraform@v3 153 | with: 154 | terraform_version: ${{ steps.setup-env.outputs.terraform-version }} 155 | - name: Install go-critic 156 | env: 157 | PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic 158 | PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} 159 | run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} 160 | - name: Install goimports 161 | env: 162 | PACKAGE_URL: golang.org/x/tools/cmd/goimports 163 | PACKAGE_VERSION: ${{ steps.setup-env.outputs.goimports-version }} 164 | run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} 165 | - name: Install gosec 166 | env: 167 | PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec 168 | PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} 169 | run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} 170 | - name: Install staticcheck 171 | env: 172 | PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck 173 | PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} 174 | run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} 175 | # TODO: https://github.com/cisagov/skeleton-generic/issues/165 176 | # We are temporarily using @mcdonnnj's forked branch of terraform-docs 177 | # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 178 | # is approved. This temporary fix will allow for ATX header support when 179 | # terraform-docs is run during linting. 180 | - name: Clone ATX headers branch from terraform-docs fork 181 | run: | 182 | git clone \ 183 | --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ 184 | --depth $TERRAFORM_DOCS_REPO_DEPTH \ 185 | --single-branch \ 186 | $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs 187 | - name: Build and install terraform-docs binary 188 | run: | 189 | go build \ 190 | -C /tmp/terraform-docs \ 191 | -o $(go env GOPATH)/bin/terraform-docs 192 | - name: Install dependencies 193 | run: | 194 | python -m pip install --upgrade pip setuptools wheel 195 | pip install --upgrade --requirement requirements-test.txt 196 | - name: Set up pre-commit hook environments 197 | run: pre-commit install-hooks 198 | - name: Run pre-commit on all files 199 | run: pre-commit run --all-files 200 | - name: Setup tmate debug session 201 | uses: mxschmitt/action-tmate@v3 202 | if: env.RUN_TMATE 203 | test: 204 | name: test source - py${{ matrix.python-version }} 205 | needs: 206 | - diagnostics 207 | permissions: 208 | # actions/checkout needs this to fetch code 209 | contents: read 210 | runs-on: ${{ matrix.os }} 211 | strategy: 212 | fail-fast: false 213 | matrix: 214 | include: 215 | - os: ubuntu-22.04 216 | python-version: "3.7" 217 | os: 218 | - ubuntu-latest 219 | python-version: 220 | - "3.8" 221 | - "3.9" 222 | - "3.10" 223 | - "3.11" 224 | - "3.12" 225 | - "3.13" 226 | steps: 227 | - name: Apply standard cisagov job preamble 228 | uses: cisagov/action-job-preamble@v1 229 | with: 230 | # This functionality is poorly implemented and has been 231 | # causing problems due to the MITM implementation hogging or 232 | # leaking memory. As a result we disable it by default. If 233 | # you want to temporarily enable it, simply set 234 | # monitor_permissions equal to "true". 235 | # 236 | # TODO: Re-enable this functionality when practical. See 237 | # cisagov/skeleton-python-library#149 for more details. 238 | monitor_permissions: "false" 239 | # Use a variable to specify the permissions monitoring 240 | # configuration. By default this will yield the 241 | # configuration stored in the cisagov organization-level 242 | # variable, but if you want to use a different configuration 243 | # then simply: 244 | # 1. Create a repository-level variable with the name 245 | # ACTIONS_PERMISSIONS_CONFIG. 246 | # 2. Set this new variable's value to the configuration you 247 | # want to use for this repository. 248 | # 249 | # Note in particular that changing the permissions 250 | # monitoring configuration *does not* require you to modify 251 | # this workflow. 252 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 253 | - uses: actions/checkout@v4 254 | - id: setup-python 255 | uses: actions/setup-python@v5 256 | with: 257 | python-version: ${{ matrix.python-version }} 258 | - uses: actions/cache@v4 259 | env: 260 | BASE_CACHE_KEY: ${{ github.job }}-${{ runner.os }}-\ 261 | py${{ steps.setup-python.outputs.python-version }}- 262 | with: 263 | path: ${{ env.PIP_CACHE_DIR }} 264 | # We do not use '**/setup.py' in the cache key so only the 'setup.py' 265 | # file in the root of the repository is used. This is in case a Python 266 | # package were to have a 'setup.py' as part of its internal codebase. 267 | key: ${{ env.BASE_CACHE_KEY }}\ 268 | ${{ hashFiles('**/requirements-test.txt') }}-\ 269 | ${{ hashFiles('**/requirements.txt') }}-\ 270 | ${{ hashFiles('setup.py') }} 271 | restore-keys: | 272 | ${{ env.BASE_CACHE_KEY }} 273 | - name: Install dependencies 274 | run: | 275 | python -m pip install --upgrade pip 276 | pip install --upgrade --requirement requirements-test.txt 277 | - name: Run tests 278 | env: 279 | RELEASE_TAG: ${{ github.event.release.tag_name }} 280 | run: pytest 281 | - name: Upload coverage report 282 | uses: coverallsapp/github-action@v2 283 | with: 284 | flag-name: py${{ matrix.python-version }} 285 | parallel: true 286 | if: success() 287 | - name: Setup tmate debug session 288 | uses: mxschmitt/action-tmate@v3 289 | if: env.RUN_TMATE 290 | coveralls-finish: 291 | permissions: 292 | # actions/checkout needs this to fetch code 293 | contents: read 294 | runs-on: ubuntu-latest 295 | needs: 296 | - diagnostics 297 | - test 298 | steps: 299 | - name: Apply standard cisagov job preamble 300 | uses: cisagov/action-job-preamble@v1 301 | with: 302 | # This functionality is poorly implemented and has been 303 | # causing problems due to the MITM implementation hogging or 304 | # leaking memory. As a result we disable it by default. If 305 | # you want to temporarily enable it, simply set 306 | # monitor_permissions equal to "true". 307 | # 308 | # TODO: Re-enable this functionality when practical. See 309 | # cisagov/skeleton-python-library#149 for more details. 310 | monitor_permissions: "false" 311 | # Use a variable to specify the permissions monitoring 312 | # configuration. By default this will yield the 313 | # configuration stored in the cisagov organization-level 314 | # variable, but if you want to use a different configuration 315 | # then simply: 316 | # 1. Create a repository-level variable with the name 317 | # ACTIONS_PERMISSIONS_CONFIG. 318 | # 2. Set this new variable's value to the configuration you 319 | # want to use for this repository. 320 | # 321 | # Note in particular that changing the permissions 322 | # monitoring configuration *does not* require you to modify 323 | # this workflow. 324 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 325 | - uses: actions/checkout@v4 326 | - name: Finished coveralls reports 327 | uses: coverallsapp/github-action@v2 328 | with: 329 | parallel-finished: true 330 | - name: Setup tmate debug session 331 | uses: mxschmitt/action-tmate@v3 332 | if: env.RUN_TMATE 333 | build: 334 | name: build wheel - py${{ matrix.python-version }} 335 | needs: 336 | - diagnostics 337 | - lint 338 | - test 339 | permissions: 340 | # actions/checkout needs this to fetch code 341 | contents: read 342 | runs-on: ${{ matrix.os }} 343 | strategy: 344 | fail-fast: false 345 | matrix: 346 | include: 347 | - os: ubuntu-22.04 348 | python-version: "3.7" 349 | os: 350 | - ubuntu-latest 351 | python-version: 352 | - "3.8" 353 | - "3.9" 354 | - "3.10" 355 | - "3.11" 356 | - "3.12" 357 | - "3.13" 358 | steps: 359 | - name: Apply standard cisagov job preamble 360 | uses: cisagov/action-job-preamble@v1 361 | with: 362 | # This functionality is poorly implemented and has been 363 | # causing problems due to the MITM implementation hogging or 364 | # leaking memory. As a result we disable it by default. If 365 | # you want to temporarily enable it, simply set 366 | # monitor_permissions equal to "true". 367 | # 368 | # TODO: Re-enable this functionality when practical. See 369 | # cisagov/skeleton-python-library#149 for more details. 370 | monitor_permissions: "false" 371 | # Use a variable to specify the permissions monitoring 372 | # configuration. By default this will yield the 373 | # configuration stored in the cisagov organization-level 374 | # variable, but if you want to use a different configuration 375 | # then simply: 376 | # 1. Create a repository-level variable with the name 377 | # ACTIONS_PERMISSIONS_CONFIG. 378 | # 2. Set this new variable's value to the configuration you 379 | # want to use for this repository. 380 | # 381 | # Note in particular that changing the permissions 382 | # monitoring configuration *does not* require you to modify 383 | # this workflow. 384 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 385 | - uses: actions/checkout@v4 386 | - id: setup-python 387 | uses: actions/setup-python@v5 388 | with: 389 | python-version: ${{ matrix.python-version }} 390 | - uses: actions/cache@v4 391 | env: 392 | BASE_CACHE_KEY: ${{ github.job }}-${{ runner.os }}-\ 393 | py${{ steps.setup-python.outputs.python-version }}- 394 | with: 395 | path: ${{ env.PIP_CACHE_DIR }} 396 | # We do not use '**/setup.py' in the cache key so only the 'setup.py' 397 | # file in the root of the repository is used. This is in case a Python 398 | # package were to have a 'setup.py' as part of its internal codebase. 399 | key: ${{ env.BASE_CACHE_KEY }}\ 400 | ${{ hashFiles('**/requirements.txt') }}-\ 401 | ${{ hashFiles('setup.py') }} 402 | restore-keys: | 403 | ${{ env.BASE_CACHE_KEY }} 404 | - name: Install build dependencies 405 | run: | 406 | python -m pip install --upgrade pip setuptools wheel 407 | python -m pip install --upgrade build 408 | - name: Build artifacts 409 | run: python -m build 410 | - name: Upload artifacts 411 | uses: actions/upload-artifact@v4 412 | with: 413 | name: dist-${{ matrix.python-version }} 414 | path: dist 415 | - name: Setup tmate debug session 416 | uses: mxschmitt/action-tmate@v3 417 | if: env.RUN_TMATE 418 | test-build: 419 | name: test built wheel - py${{ matrix.python-version }} 420 | needs: 421 | - diagnostics 422 | - build 423 | permissions: 424 | # actions/checkout needs this to fetch code 425 | contents: read 426 | runs-on: ${{ matrix.os }} 427 | strategy: 428 | fail-fast: false 429 | matrix: 430 | include: 431 | - os: ubuntu-22.04 432 | python-version: "3.7" 433 | os: 434 | - ubuntu-latest 435 | python-version: 436 | - "3.8" 437 | - "3.9" 438 | - "3.10" 439 | - "3.11" 440 | - "3.12" 441 | - "3.13" 442 | steps: 443 | - name: Apply standard cisagov job preamble 444 | uses: cisagov/action-job-preamble@v1 445 | with: 446 | # This functionality is poorly implemented and has been 447 | # causing problems due to the MITM implementation hogging or 448 | # leaking memory. As a result we disable it by default. If 449 | # you want to temporarily enable it, simply set 450 | # monitor_permissions equal to "true". 451 | # 452 | # TODO: Re-enable this functionality when practical. See 453 | # cisagov/skeleton-python-library#149 for more details. 454 | monitor_permissions: "false" 455 | # Use a variable to specify the permissions monitoring 456 | # configuration. By default this will yield the 457 | # configuration stored in the cisagov organization-level 458 | # variable, but if you want to use a different configuration 459 | # then simply: 460 | # 1. Create a repository-level variable with the name 461 | # ACTIONS_PERMISSIONS_CONFIG. 462 | # 2. Set this new variable's value to the configuration you 463 | # want to use for this repository. 464 | # 465 | # Note in particular that changing the permissions 466 | # monitoring configuration *does not* require you to modify 467 | # this workflow. 468 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 469 | - uses: actions/checkout@v4 470 | - id: setup-python 471 | uses: actions/setup-python@v5 472 | with: 473 | python-version: ${{ matrix.python-version }} 474 | - uses: actions/cache@v4 475 | env: 476 | BASE_CACHE_KEY: ${{ github.job }}-${{ runner.os }}-\ 477 | py${{ steps.setup-python.outputs.python-version }}- 478 | with: 479 | path: ${{ env.PIP_CACHE_DIR }} 480 | # We do not use '**/setup.py' in the cache key so only the 'setup.py' 481 | # file in the root of the repository is used. This is in case a Python 482 | # package were to have a 'setup.py' as part of its internal codebase. 483 | key: ${{ env.BASE_CACHE_KEY }}\ 484 | ${{ hashFiles('**/requirements.txt') }}-\ 485 | ${{ hashFiles('setup.py') }} 486 | restore-keys: | 487 | ${{ env.BASE_CACHE_KEY }} 488 | - name: Retrieve the built wheel 489 | uses: actions/download-artifact@v4 490 | with: 491 | name: dist-${{ matrix.python-version }} 492 | path: dist 493 | - id: find-wheel 494 | name: Get the name of the retrieved wheel (there should only be one) 495 | run: echo "wheel=$(ls dist/*whl)" >> $GITHUB_OUTPUT 496 | - name: Update core Python packages 497 | run: python -m pip install --upgrade pip setuptools wheel 498 | - name: Install the built wheel (along with testing dependencies) 499 | run: python -m pip install ${{ steps.find-wheel.outputs.wheel }}[test] 500 | - name: Run tests 501 | env: 502 | RELEASE_TAG: ${{ github.event.release.tag_name }} 503 | run: pytest 504 | - name: Setup tmate debug session 505 | uses: mxschmitt/action-tmate@v3 506 | if: env.RUN_TMATE 507 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # For most projects, this workflow file will not need changing; you simply need 3 | # to commit it to your repository. 4 | # 5 | # You may wish to alter this file to override the set of languages analyzed, 6 | # or to provide custom queries or build logic. 7 | name: CodeQL 8 | 9 | # The use of on here as a key is part of the GitHub actions syntax. 10 | # yamllint disable-line rule:truthy 11 | on: 12 | merge_group: 13 | types: 14 | - checks_requested 15 | pull_request: 16 | # The branches here must be a subset of the ones in the push key 17 | branches: 18 | - develop 19 | push: 20 | # Dependabot-triggered push events have read-only access, but uploading code 21 | # scanning requires write access. 22 | branches-ignore: 23 | - dependabot/** 24 | schedule: 25 | - cron: 0 14 * * 6 26 | 27 | jobs: 28 | diagnostics: 29 | name: Run diagnostics 30 | # This job does not need any permissions 31 | permissions: {} 32 | runs-on: ubuntu-latest 33 | steps: 34 | # Note that a duplicate of this step must be added at the top of 35 | # each job. 36 | - name: Apply standard cisagov job preamble 37 | uses: cisagov/action-job-preamble@v1 38 | with: 39 | check_github_status: "true" 40 | # This functionality is poorly implemented and has been 41 | # causing problems due to the MITM implementation hogging or 42 | # leaking memory. As a result we disable it by default. If 43 | # you want to temporarily enable it, simply set 44 | # monitor_permissions equal to "true". 45 | # 46 | # TODO: Re-enable this functionality when practical. See 47 | # cisagov/skeleton-generic#207 for more details. 48 | monitor_permissions: "false" 49 | output_workflow_context: "true" 50 | # Use a variable to specify the permissions monitoring 51 | # configuration. By default this will yield the 52 | # configuration stored in the cisagov organization-level 53 | # variable, but if you want to use a different configuration 54 | # then simply: 55 | # 1. Create a repository-level variable with the name 56 | # ACTIONS_PERMISSIONS_CONFIG. 57 | # 2. Set this new variable's value to the configuration you 58 | # want to use for this repository. 59 | # 60 | # Note in particular that changing the permissions 61 | # monitoring configuration *does not* require you to modify 62 | # this workflow. 63 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 64 | analyze: 65 | name: Analyze 66 | needs: 67 | - diagnostics 68 | permissions: 69 | # actions/checkout needs this to fetch code 70 | contents: read 71 | # required for all workflows 72 | security-events: write 73 | runs-on: ubuntu-latest 74 | strategy: 75 | fail-fast: false 76 | matrix: 77 | # Override automatic language detection by changing the below 78 | # list 79 | # 80 | # Supported options are actions, c-cpp, csharp, go, 81 | # java-kotlin, javascript-typescript, python, ruby, and swift. 82 | language: 83 | - actions 84 | - python 85 | # Learn more... 86 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 87 | 88 | steps: 89 | - name: Apply standard cisagov job preamble 90 | uses: cisagov/action-job-preamble@v1 91 | with: 92 | # This functionality is poorly implemented and has been 93 | # causing problems due to the MITM implementation hogging or 94 | # leaking memory. As a result we disable it by default. If 95 | # you want to temporarily enable it, simply set 96 | # monitor_permissions equal to "true". 97 | # 98 | # TODO: Re-enable this functionality when practical. See 99 | # cisagov/skeleton-generic#207 for more details. 100 | monitor_permissions: "false" 101 | # Use a variable to specify the permissions monitoring 102 | # configuration. By default this will yield the 103 | # configuration stored in the cisagov organization-level 104 | # variable, but if you want to use a different configuration 105 | # then simply: 106 | # 1. Create a repository-level variable with the name 107 | # ACTIONS_PERMISSIONS_CONFIG. 108 | # 2. Set this new variable's value to the configuration you 109 | # want to use for this repository. 110 | # 111 | # Note in particular that changing the permissions 112 | # monitoring configuration *does not* require you to modify 113 | # this workflow. 114 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 115 | 116 | - name: Checkout repository 117 | uses: actions/checkout@v4 118 | 119 | # Initializes the CodeQL tools for scanning. 120 | - name: Initialize CodeQL 121 | uses: github/codeql-action/init@v3 122 | with: 123 | languages: ${{ matrix.language }} 124 | 125 | # Autobuild attempts to build any compiled languages (C/C++, C#, or 126 | # Java). If this step fails, then you should remove it and run the build 127 | # manually (see below). 128 | - name: Autobuild 129 | uses: github/codeql-action/autobuild@v3 130 | 131 | # ℹ️ Command-line programs to run using the OS shell. 132 | # 📚 https://git.io/JvXDl 133 | 134 | # ✏️ If the Autobuild fails above, remove it and uncomment the following 135 | # three lines and modify them (or add more) to build your code if your 136 | # project uses a compiled language 137 | 138 | # - run: | 139 | # make bootstrap 140 | # make release 141 | 142 | - name: Perform CodeQL Analysis 143 | uses: github/codeql-action/analyze@v3 144 | -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Dependency review 3 | 4 | on: # yamllint disable-line rule:truthy 5 | merge_group: 6 | types: 7 | - checks_requested 8 | pull_request: 9 | 10 | # Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, 11 | # nounset, errexit, and pipefail. The `-x` will print all commands as they are 12 | # run. Please see the GitHub Actions documentation for more information: 13 | # https://docs.github.com/en/actions/using-jobs/setting-default-values-for-jobs 14 | defaults: 15 | run: 16 | shell: bash -Eueo pipefail -x {0} 17 | 18 | jobs: 19 | diagnostics: 20 | name: Run diagnostics 21 | # This job does not need any permissions 22 | permissions: {} 23 | runs-on: ubuntu-latest 24 | steps: 25 | # Note that a duplicate of this step must be added at the top of 26 | # each job. 27 | - name: Apply standard cisagov job preamble 28 | uses: cisagov/action-job-preamble@v1 29 | with: 30 | check_github_status: "true" 31 | # This functionality is poorly implemented and has been 32 | # causing problems due to the MITM implementation hogging or 33 | # leaking memory. As a result we disable it by default. If 34 | # you want to temporarily enable it, simply set 35 | # monitor_permissions equal to "true". 36 | # 37 | # TODO: Re-enable this functionality when practical. See 38 | # cisagov/skeleton-generic#207 for more details. 39 | monitor_permissions: "false" 40 | output_workflow_context: "true" 41 | # Use a variable to specify the permissions monitoring 42 | # configuration. By default this will yield the 43 | # configuration stored in the cisagov organization-level 44 | # variable, but if you want to use a different configuration 45 | # then simply: 46 | # 1. Create a repository-level variable with the name 47 | # ACTIONS_PERMISSIONS_CONFIG. 48 | # 2. Set this new variable's value to the configuration you 49 | # want to use for this repository. 50 | # 51 | # Note in particular that changing the permissions 52 | # monitoring configuration *does not* require you to modify 53 | # this workflow. 54 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 55 | dependency-review: 56 | name: Dependency review 57 | needs: 58 | - diagnostics 59 | permissions: 60 | # actions/checkout needs this to fetch code 61 | contents: read 62 | runs-on: ubuntu-latest 63 | steps: 64 | - name: Apply standard cisagov job preamble 65 | uses: cisagov/action-job-preamble@v1 66 | with: 67 | # This functionality is poorly implemented and has been 68 | # causing problems due to the MITM implementation hogging or 69 | # leaking memory. As a result we disable it by default. If 70 | # you want to temporarily enable it, simply set 71 | # monitor_permissions equal to "true". 72 | # 73 | # TODO: Re-enable this functionality when practical. See 74 | # cisagov/skeleton-generic#207 for more details. 75 | monitor_permissions: "false" 76 | # Use a variable to specify the permissions monitoring 77 | # configuration. By default this will yield the 78 | # configuration stored in the cisagov organization-level 79 | # variable, but if you want to use a different configuration 80 | # then simply: 81 | # 1. Create a repository-level variable with the name 82 | # ACTIONS_PERMISSIONS_CONFIG. 83 | # 2. Set this new variable's value to the configuration you 84 | # want to use for this repository. 85 | # 86 | # Note in particular that changing the permissions 87 | # monitoring configuration *does not* require you to modify 88 | # this workflow. 89 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 90 | - id: checkout-repo 91 | name: Checkout the repository 92 | uses: actions/checkout@v4 93 | - id: dependency-review 94 | name: Review dependency changes for vulnerabilities and license changes 95 | uses: actions/dependency-review-action@v4 96 | -------------------------------------------------------------------------------- /.github/workflows/sync-labels.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: sync-labels 3 | 4 | on: # yamllint disable-line rule:truthy 5 | push: 6 | paths: 7 | - .github/labels.yml 8 | - .github/workflows/sync-labels.yml 9 | workflow_dispatch: 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | diagnostics: 16 | name: Run diagnostics 17 | # This job does not need any permissions 18 | permissions: {} 19 | runs-on: ubuntu-latest 20 | steps: 21 | # Note that a duplicate of this step must be added at the top of 22 | # each job. 23 | - name: Apply standard cisagov job preamble 24 | uses: cisagov/action-job-preamble@v1 25 | with: 26 | check_github_status: "true" 27 | # This functionality is poorly implemented and has been 28 | # causing problems due to the MITM implementation hogging or 29 | # leaking memory. As a result we disable it by default. If 30 | # you want to temporarily enable it, simply set 31 | # monitor_permissions equal to "true". 32 | # 33 | # TODO: Re-enable this functionality when practical. See 34 | # cisagov/skeleton-generic#207 for more details. 35 | monitor_permissions: "false" 36 | output_workflow_context: "true" 37 | # Use a variable to specify the permissions monitoring 38 | # configuration. By default this will yield the 39 | # configuration stored in the cisagov organization-level 40 | # variable, but if you want to use a different configuration 41 | # then simply: 42 | # 1. Create a repository-level variable with the name 43 | # ACTIONS_PERMISSIONS_CONFIG. 44 | # 2. Set this new variable's value to the configuration you 45 | # want to use for this repository. 46 | # 47 | # Note in particular that changing the permissions 48 | # monitoring configuration *does not* require you to modify 49 | # this workflow. 50 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 51 | labeler: 52 | needs: 53 | - diagnostics 54 | permissions: 55 | # actions/checkout needs this to fetch code 56 | contents: read 57 | # crazy-max/ghaction-github-labeler needs this to manage repository labels 58 | issues: write 59 | runs-on: ubuntu-latest 60 | steps: 61 | - name: Apply standard cisagov job preamble 62 | uses: cisagov/action-job-preamble@v1 63 | with: 64 | # This functionality is poorly implemented and has been 65 | # causing problems due to the MITM implementation hogging or 66 | # leaking memory. As a result we disable it by default. If 67 | # you want to temporarily enable it, simply set 68 | # monitor_permissions equal to "true". 69 | # 70 | # TODO: Re-enable this functionality when practical. See 71 | # cisagov/skeleton-generic#207 for more details. 72 | monitor_permissions: "false" 73 | # Use a variable to specify the permissions monitoring 74 | # configuration. By default this will yield the 75 | # configuration stored in the cisagov organization-level 76 | # variable, but if you want to use a different configuration 77 | # then simply: 78 | # 1. Create a repository-level variable with the name 79 | # ACTIONS_PERMISSIONS_CONFIG. 80 | # 2. Set this new variable's value to the configuration you 81 | # want to use for this repository. 82 | # 83 | # Note in particular that changing the permissions 84 | # monitoring configuration *does not* require you to modify 85 | # this workflow. 86 | permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} 87 | - uses: actions/checkout@v4 88 | - name: Sync repository labels 89 | if: success() 90 | uses: crazy-max/ghaction-github-labeler@v5 91 | with: 92 | # This is a hideous ternary equivalent so we only do a dry run unless 93 | # this workflow is triggered by the develop branch. 94 | dry-run: ${{ github.ref_name == 'develop' && 'false' || 'true' }} 95 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # This file specifies intentionally untracked files that Git should ignore. 2 | # Files already tracked by Git are not affected. 3 | # See: https://git-scm.com/docs/gitignore 4 | 5 | ## Python ## 6 | __pycache__ 7 | .coverage 8 | .mypy_cache 9 | .pytest_cache 10 | .python-version 11 | *.egg-info 12 | dist 13 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | combine_star=true 3 | force_sort_within_sections=true 4 | 5 | import_heading_stdlib=Standard Python Libraries 6 | import_heading_thirdparty=Third-Party Libraries 7 | import_heading_firstparty=cisagov Libraries 8 | 9 | # Run isort under the black profile to align with our other Python linting 10 | profile=black 11 | -------------------------------------------------------------------------------- /.mdl_config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Default state for all rules 4 | default: true 5 | 6 | # MD003/heading-style/header-style - Heading style 7 | MD003: 8 | # Enforce the ATX-closed style of header 9 | style: atx_closed 10 | 11 | # MD004/ul-style - Unordered list style 12 | MD004: 13 | # Enforce dashes for unordered lists 14 | style: dash 15 | 16 | # MD013/line-length - Line length 17 | MD013: 18 | # Do not enforce for code blocks 19 | code_blocks: false 20 | # Do not enforce for tables 21 | tables: false 22 | 23 | # MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the 24 | # same content 25 | MD024: 26 | # Allow headers with the same content as long as they are not in the same 27 | # parent heading 28 | allow_different_nesting: true 29 | 30 | # MD029/ol-prefix - Ordered list item prefix 31 | MD029: 32 | # Enforce the `1.` style for ordered lists 33 | style: one 34 | 35 | # MD033/no-inline-html - Inline HTML 36 | MD033: 37 | # The h1 and img elements are allowed to permit header images 38 | allowed_elements: 39 | - h1 40 | - img 41 | 42 | # MD035/hr-style - Horizontal rule style 43 | MD035: 44 | # Enforce dashes for horizontal rules 45 | style: --- 46 | 47 | # MD046/code-block-style - Code block style 48 | MD046: 49 | # Enforce the fenced style for code blocks 50 | style: fenced 51 | 52 | # MD049/emphasis-style - Emphasis style should be consistent 53 | MD049: 54 | # Enforce asterisks as the style to use for emphasis 55 | style: asterisk 56 | 57 | # MD050/strong-style - Strong style should be consistent 58 | MD050: 59 | # Enforce asterisks as the style to use for strong 60 | style: asterisk 61 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | default_language_version: 3 | # force all unspecified python hooks to run python3 4 | python: python3 5 | 6 | repos: 7 | # Check the pre-commit configuration 8 | - repo: meta 9 | hooks: 10 | - id: check-useless-excludes 11 | 12 | - repo: https://github.com/pre-commit/pre-commit-hooks 13 | rev: v5.0.0 14 | hooks: 15 | - id: check-case-conflict 16 | - id: check-executables-have-shebangs 17 | - id: check-json 18 | - id: check-merge-conflict 19 | - id: check-shebang-scripts-are-executable 20 | - id: check-symlinks 21 | - id: check-toml 22 | - id: check-vcs-permalinks 23 | - id: check-xml 24 | - id: debug-statements 25 | - id: destroyed-symlinks 26 | - id: detect-aws-credentials 27 | args: 28 | - --allow-missing-credentials 29 | - id: detect-private-key 30 | - id: end-of-file-fixer 31 | - id: mixed-line-ending 32 | args: 33 | - --fix=lf 34 | - id: pretty-format-json 35 | args: 36 | - --autofix 37 | - id: requirements-txt-fixer 38 | - id: trailing-whitespace 39 | 40 | # Text file hooks 41 | - repo: https://github.com/igorshubovych/markdownlint-cli 42 | rev: v0.44.0 43 | hooks: 44 | - id: markdownlint 45 | args: 46 | - --config=.mdl_config.yaml 47 | - repo: https://github.com/rbubley/mirrors-prettier 48 | rev: v3.5.3 49 | hooks: 50 | - id: prettier 51 | - repo: https://github.com/adrienverge/yamllint 52 | rev: v1.37.0 53 | hooks: 54 | - id: yamllint 55 | args: 56 | - --strict 57 | 58 | # GitHub Actions hooks 59 | - repo: https://github.com/python-jsonschema/check-jsonschema 60 | rev: 0.32.1 61 | hooks: 62 | - id: check-github-actions 63 | - id: check-github-workflows 64 | 65 | # pre-commit hooks 66 | - repo: https://github.com/pre-commit/pre-commit 67 | rev: v4.2.0 68 | hooks: 69 | - id: validate_manifest 70 | 71 | # Go hooks 72 | - repo: https://github.com/TekWizely/pre-commit-golang 73 | rev: v1.0.0-rc.1 74 | hooks: 75 | # Go Build 76 | - id: go-build-repo-mod 77 | # Style Checkers 78 | - id: go-critic 79 | # goimports 80 | - id: go-imports-repo 81 | args: 82 | # Write changes to files 83 | - -w 84 | # Go Mod Tidy 85 | - id: go-mod-tidy-repo 86 | # GoSec 87 | - id: go-sec-repo-mod 88 | # StaticCheck 89 | - id: go-staticcheck-repo-mod 90 | # Go Test 91 | - id: go-test-repo-mod 92 | # Go Vet 93 | - id: go-vet-repo-mod 94 | # Nix hooks 95 | - repo: https://github.com/nix-community/nixpkgs-fmt 96 | rev: v1.3.0 97 | hooks: 98 | - id: nixpkgs-fmt 99 | 100 | # Shell script hooks 101 | - repo: https://github.com/scop/pre-commit-shfmt 102 | rev: v3.11.0-1 103 | hooks: 104 | - id: shfmt 105 | args: 106 | # List files that will be formatted 107 | - --list 108 | # Write result to file instead of stdout 109 | - --write 110 | # Indent by two spaces 111 | - --indent 112 | - "2" 113 | # Binary operators may start a line 114 | - --binary-next-line 115 | # Switch cases are indented 116 | - --case-indent 117 | # Redirect operators are followed by a space 118 | - --space-redirects 119 | - repo: https://github.com/shellcheck-py/shellcheck-py 120 | rev: v0.10.0.1 121 | hooks: 122 | - id: shellcheck 123 | 124 | # Python hooks 125 | # Run bandit on the "tests" tree with a configuration 126 | - repo: https://github.com/PyCQA/bandit 127 | rev: 1.8.3 128 | hooks: 129 | - id: bandit 130 | name: bandit (tests tree) 131 | files: tests 132 | args: 133 | - --config=.bandit.yml 134 | # Run bandit on everything except the "tests" tree 135 | - repo: https://github.com/PyCQA/bandit 136 | rev: 1.8.3 137 | hooks: 138 | - id: bandit 139 | name: bandit (everything else) 140 | exclude: tests 141 | - repo: https://github.com/psf/black-pre-commit-mirror 142 | rev: 25.1.0 143 | hooks: 144 | - id: black 145 | - repo: https://github.com/PyCQA/flake8 146 | rev: 7.1.2 147 | hooks: 148 | - id: flake8 149 | additional_dependencies: 150 | - flake8-docstrings==1.7.0 151 | - repo: https://github.com/PyCQA/isort 152 | rev: 6.0.1 153 | hooks: 154 | - id: isort 155 | - repo: https://github.com/pre-commit/mirrors-mypy 156 | rev: v1.15.0 157 | hooks: 158 | - id: mypy 159 | # IMPORTANT: Keep type hinting-related dependencies of the 160 | # mypy pre-commit hook additional_dependencies in sync with 161 | # the dev section of setup.py to avoid discrepancies in type 162 | # checking between environments. 163 | additional_dependencies: 164 | - types-docopt 165 | - types-requests 166 | - types-setuptools 167 | - repo: https://github.com/pypa/pip-audit 168 | rev: v2.8.0 169 | hooks: 170 | - id: pip-audit 171 | args: 172 | # Add any pip requirements files to scan 173 | - --requirement 174 | - requirements-dev.txt 175 | - --requirement 176 | - requirements-test.txt 177 | - --requirement 178 | - requirements.txt 179 | - repo: https://github.com/asottile/pyupgrade 180 | rev: v3.19.1 181 | hooks: 182 | - id: pyupgrade 183 | 184 | # Ansible hooks 185 | - repo: https://github.com/ansible/ansible-lint 186 | rev: v25.1.3 187 | hooks: 188 | - id: ansible-lint 189 | additional_dependencies: 190 | # On its own ansible-lint does not pull in ansible, only 191 | # ansible-core. Therefore, if an Ansible module lives in 192 | # ansible instead of ansible-core, the linter will complain 193 | # that the module is unknown. In these cases it is 194 | # necessary to add the ansible package itself as an 195 | # additional dependency, with the same pinning as is done in 196 | # requirements-test.txt of cisagov/skeleton-ansible-role. 197 | # 198 | # Version 10 is required because the pip-audit pre-commit 199 | # hook identifies a vulnerability in ansible-core 2.16.13, 200 | # but all versions of ansible 9 have a dependency on 201 | # ~=2.16.X. 202 | # 203 | # It is also a good idea to go ahead and upgrade to version 204 | # 10 since version 9 is going EOL at the end of November: 205 | # https://endoflife.date/ansible 206 | # - ansible>=10,<11 207 | # ansible-core 2.16.3 through 2.16.6 suffer from the bug 208 | # discussed in ansible/ansible#82702, which breaks any 209 | # symlinked files in vars, tasks, etc. for any Ansible role 210 | # installed via ansible-galaxy. Hence we never want to 211 | # install those versions. 212 | # 213 | # Note that the pip-audit pre-commit hook identifies a 214 | # vulnerability in ansible-core 2.16.13. The pin of 215 | # ansible-core to >=2.17 effectively also pins ansible to 216 | # >=10. 217 | # 218 | # It is also a good idea to go ahead and upgrade to 219 | # ansible-core 2.17 since security support for ansible-core 220 | # 2.16 ends this month: 221 | # https://docs.ansible.com/ansible/devel/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix 222 | # 223 | # Note that any changes made to this dependency must also be 224 | # made in requirements.txt in cisagov/skeleton-packer and 225 | # requirements-test.txt in cisagov/skeleton-ansible-role. 226 | - ansible-core>=2.17 227 | 228 | # Terraform hooks 229 | - repo: https://github.com/antonbabenko/pre-commit-terraform 230 | rev: v1.98.0 231 | hooks: 232 | - id: terraform_fmt 233 | - id: terraform_validate 234 | 235 | # Docker hooks 236 | - repo: https://github.com/IamTheFij/docker-pre-commit 237 | rev: v3.0.1 238 | hooks: 239 | - id: docker-compose-check 240 | 241 | # Packer hooks 242 | - repo: https://github.com/cisagov/pre-commit-packer 243 | rev: v0.3.0 244 | hooks: 245 | - id: packer_fmt 246 | - id: packer_validate 247 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # Already being linted by pretty-format-json 2 | *.json 3 | # Already being linted by mdl 4 | *.md 5 | # Already being linted by yamllint 6 | *.yaml 7 | *.yml 8 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | rules: 5 | braces: 6 | # Do not allow non-empty flow mappings 7 | forbid: non-empty 8 | # Allow up to one space inside braces. This is required for Ansible compatibility. 9 | max-spaces-inside: 1 10 | 11 | brackets: 12 | # Do not allow non-empty flow sequences 13 | forbid: non-empty 14 | 15 | comments: 16 | # Ensure that inline comments have at least one space before the preceding content. 17 | # This is required for Ansible compatibility. 18 | min-spaces-from-content: 1 19 | 20 | # yamllint does not like it when you comment out different parts of 21 | # dictionaries in a list. You can see 22 | # https://github.com/adrienverge/yamllint/issues/384 for some examples of 23 | # this behavior. 24 | comments-indentation: disable 25 | 26 | indentation: 27 | # Ensure that block sequences inside of a mapping are indented 28 | indent-sequences: true 29 | # Enforce a specific number of spaces 30 | spaces: 2 31 | 32 | # yamllint does not allow inline mappings that exceed the line length by 33 | # default. There are many scenarios where the inline mapping may be a key, 34 | # hash, or other long value that would exceed the line length but cannot 35 | # reasonably be broken across lines. 36 | line-length: 37 | # This rule implies the allow-non-breakable-words rule 38 | allow-non-breakable-inline-mappings: true 39 | # Allows a 10% overage from the default limit of 80 40 | max: 88 41 | 42 | # Using anything other than strings to express octal values can lead to unexpected 43 | # and potentially unsafe behavior. Ansible strongly recommends against such practices 44 | # and these rules are needed for Ansible compatibility. Please see the following for 45 | # more information: 46 | # https://ansible.readthedocs.io/projects/lint/rules/risky-octal/ 47 | octal-values: 48 | # Do not allow explicit octal values (those beginning with a leading 0o). 49 | forbid-explicit-octal: true 50 | # Do not allow implicit octal values (those beginning with a leading 0). 51 | forbid-implicit-octal: true 52 | 53 | quoted-strings: 54 | # Allow disallowed quotes (single quotes) for strings that contain allowed quotes 55 | # (double quotes). 56 | allow-quoted-quotes: true 57 | # Apply these rules to keys in mappings as well 58 | check-keys: true 59 | # We prefer double quotes for strings when they are needed 60 | quote-type: double 61 | # Only require quotes when they are necessary for proper processing 62 | required: only-when-needed 63 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Welcome # 2 | 3 | We're so glad you're thinking about contributing to this open source 4 | project! If you're unsure or afraid of anything, just ask or submit 5 | the issue or pull request anyway. The worst that can happen is that 6 | you'll be politely asked to change something. We appreciate any sort 7 | of contribution, and don't want a wall of rules to get in the way of 8 | that. 9 | 10 | Before contributing, we encourage you to read our CONTRIBUTING policy 11 | (you are here), our [LICENSE](LICENSE), and our [README](README.md), 12 | all of which should be in this repository. 13 | 14 | ## Issues ## 15 | 16 | If you want to report a bug or request a new feature, the most direct 17 | method is to [create an 18 | issue](https://github.com/cisagov/trustymail/issues) in 19 | this repository. We recommend that you first search through existing 20 | issues (both open and closed) to check if your particular issue has 21 | already been reported. If it has then you might want to add a comment 22 | to the existing issue. If it hasn't then feel free to create a new 23 | one. 24 | 25 | ## Pull requests ## 26 | 27 | If you choose to [submit a pull 28 | request](https://github.com/cisagov/trustymail/pulls), 29 | you will notice that our continuous integration (CI) system runs a 30 | fairly extensive set of linters, syntax checkers, system, and unit tests. 31 | Your pull request may fail these checks, and that's OK. If you want 32 | you can stop there and wait for us to make the necessary corrections 33 | to ensure your code passes the CI checks. 34 | 35 | If you want to make the changes yourself, or if you want to become a 36 | regular contributor, then you will want to set up 37 | [pre-commit](https://pre-commit.com/) on your local machine. Once you 38 | do that, the CI checks will run locally before you even write your 39 | commit message. This speeds up your development cycle considerably. 40 | 41 | ### Setting up pre-commit ### 42 | 43 | There are a few ways to do this, but we prefer to use 44 | [`pyenv`](https://github.com/pyenv/pyenv) and 45 | [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) to 46 | create and manage a Python virtual environment specific to this 47 | project. 48 | 49 | We recommend using the `setup-env` script located in this repository, 50 | as it automates the entire environment configuration process. The 51 | dependencies required to run this script are 52 | [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), 53 | [`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). 54 | If these tools are already configured on your system, you can simply run the 55 | following command: 56 | 57 | ```console 58 | ./setup-env 59 | ``` 60 | 61 | Otherwise, follow the steps below to manually configure your 62 | environment. 63 | 64 | #### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### 65 | 66 | On macOS, we recommend installing [brew](https://brew.sh/). Then 67 | installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and 68 | adding this to your profile: 69 | 70 | ```bash 71 | # GNU getopt must be explicitly added to the path since it is 72 | # keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean) 73 | export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" 74 | 75 | # Setup pyenv 76 | export PYENV_ROOT="$HOME/.pyenv" 77 | export PATH="$PYENV_ROOT/bin:$PATH" 78 | eval "$(pyenv init --path)" 79 | eval "$(pyenv init -)" 80 | eval "$(pyenv virtualenv-init -)" 81 | ``` 82 | 83 | For Linux, Windows Subsystem for Linux (WSL), or macOS (if you 84 | don't want to use `brew`) you can use 85 | [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to 86 | install the necessary tools. Before running this ensure that you have 87 | installed the prerequisites for your platform according to the 88 | [`pyenv` wiki 89 | page](https://github.com/pyenv/pyenv/wiki/common-build-problems). 90 | GNU `getopt` is included in most Linux distributions as part of the 91 | [`util-linux`](https://github.com/util-linux/util-linux) package. 92 | 93 | On WSL you should treat your platform as whatever Linux distribution 94 | you've chosen to install. 95 | 96 | Once you have installed `pyenv` you will need to add the following 97 | lines to your `.bash_profile` (or `.profile`): 98 | 99 | ```bash 100 | export PYENV_ROOT="$HOME/.pyenv" 101 | export PATH="$PYENV_ROOT/bin:$PATH" 102 | eval "$(pyenv init --path)" 103 | ``` 104 | 105 | and then add the following lines to your `.bashrc`: 106 | 107 | ```bash 108 | eval "$(pyenv init -)" 109 | eval "$(pyenv virtualenv-init -)" 110 | ``` 111 | 112 | If you want more information about setting up `pyenv` once installed, please run 113 | 114 | ```console 115 | pyenv init 116 | ``` 117 | 118 | and 119 | 120 | ```console 121 | pyenv virtualenv-init 122 | ``` 123 | 124 | for the current configuration instructions. 125 | 126 | If you are using a shell other than `bash` you should follow the 127 | instructions that the `pyenv-installer` script outputs. 128 | 129 | You will need to reload your shell for these changes to take effect so 130 | you can begin to use `pyenv`. 131 | 132 | For a list of Python versions that are already installed and ready to 133 | use with `pyenv`, use the command `pyenv versions`. To see a list of 134 | the Python versions available to be installed and used with `pyenv` 135 | use the command `pyenv install --list`. You can read more 136 | [here](https://github.com/pyenv/pyenv/blob/master/COMMANDS.md) about 137 | the many things that `pyenv` can do. See 138 | [here](https://github.com/pyenv/pyenv-virtualenv#usage) for the 139 | additional capabilities that pyenv-virtualenv adds to the `pyenv` 140 | command. 141 | 142 | #### Creating the Python virtual environment #### 143 | 144 | Once `pyenv` and `pyenv-virtualenv` are installed on your system, you 145 | can create and configure the Python virtual environment with these 146 | commands: 147 | 148 | ```console 149 | cd trustymail 150 | pyenv virtualenv trustymail 151 | pyenv local trustymail 152 | pip install --requirement requirements-dev.txt 153 | ``` 154 | 155 | #### Installing the pre-commit hook #### 156 | 157 | Now setting up pre-commit is as simple as: 158 | 159 | ```console 160 | pre-commit install 161 | ``` 162 | 163 | At this point the pre-commit checks will run against any files that 164 | you attempt to commit. If you want to run the checks against the 165 | entire repo, just execute `pre-commit run --all-files`. 166 | 167 | ### Running unit and system tests ### 168 | 169 | In addition to the pre-commit checks the CI system will run the suite 170 | of unit and system tests that are included with this project. To run 171 | these tests locally execute `pytest` from the root of the project. 172 | 173 | We encourage any updates to these tests to improve the overall code 174 | coverage. If your pull request adds new functionality we would 175 | appreciate it if you extend existing test cases, or add new ones to 176 | exercise the newly added code. 177 | 178 | ## Public domain ## 179 | 180 | This project is in the public domain within the United States, and 181 | copyright and related rights in the work worldwide are waived through 182 | the [CC0 1.0 Universal public domain 183 | dedication](https://creativecommons.org/publicdomain/zero/1.0/). 184 | 185 | All contributions to this project will be released under the CC0 186 | dedication. By submitting a pull request, you are agreeing to comply 187 | with this waiver of copyright interest. 188 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | 3 | WORKDIR /app 4 | 5 | RUN pip install --no-cache-dir --upgrade pip setuptools wheel 6 | 7 | COPY requirements.txt . 8 | 9 | COPY src/trustymail/ src/trustymail/ 10 | 11 | COPY README.md . 12 | 13 | COPY requirements-dev.txt . 14 | 15 | COPY setup.py . 16 | 17 | RUN pip install --no-cache-dir --requirement requirements.txt 18 | 19 | ENTRYPOINT ["trustymail"] 20 | CMD ["--help"] 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | CC0 1.0 Universal 2 | 3 | Statement of Purpose 4 | 5 | The laws of most jurisdictions throughout the world automatically confer 6 | exclusive Copyright and Related Rights (defined below) upon the creator and 7 | subsequent owner(s) (each and all, an "owner") of an original work of 8 | authorship and/or a database (each, a "Work"). 9 | 10 | Certain owners wish to permanently relinquish those rights to a Work for the 11 | purpose of contributing to a commons of creative, cultural and scientific 12 | works ("Commons") that the public can reliably and without fear of later 13 | claims of infringement build upon, modify, incorporate in other works, reuse 14 | and redistribute as freely as possible in any form whatsoever and for any 15 | purposes, including without limitation commercial purposes. These owners may 16 | contribute to the Commons to promote the ideal of a free culture and the 17 | further production of creative, cultural and scientific works, or to gain 18 | reputation or greater distribution for their Work in part through the use and 19 | efforts of others. 20 | 21 | For these and/or other purposes and motivations, and without any expectation 22 | of additional consideration or compensation, the person associating CC0 with a 23 | Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 24 | and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 25 | and publicly distribute the Work under its terms, with knowledge of his or her 26 | Copyright and Related Rights in the Work and the meaning and intended legal 27 | effect of CC0 on those rights. 28 | 29 | 1. Copyright and Related Rights. A Work made available under CC0 may be 30 | protected by copyright and related or neighboring rights ("Copyright and 31 | Related Rights"). Copyright and Related Rights include, but are not limited 32 | to, the following: 33 | 34 | i. the right to reproduce, adapt, distribute, perform, display, communicate, 35 | and translate a Work; 36 | 37 | ii. moral rights retained by the original author(s) and/or performer(s); 38 | 39 | iii. publicity and privacy rights pertaining to a person's image or likeness 40 | depicted in a Work; 41 | 42 | iv. rights protecting against unfair competition in regards to a Work, 43 | subject to the limitations in paragraph 4(a), below; 44 | 45 | v. rights protecting the extraction, dissemination, use and reuse of data in 46 | a Work; 47 | 48 | vi. database rights (such as those arising under Directive 96/9/EC of the 49 | European Parliament and of the Council of 11 March 1996 on the legal 50 | protection of databases, and under any national implementation thereof, 51 | including any amended or successor version of such directive); and 52 | 53 | vii. other similar, equivalent or corresponding rights throughout the world 54 | based on applicable law or treaty, and any national implementations thereof. 55 | 56 | 2. Waiver. To the greatest extent permitted by, but not in contravention of, 57 | applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 58 | unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 59 | and Related Rights and associated claims and causes of action, whether now 60 | known or unknown (including existing as well as future claims and causes of 61 | action), in the Work (i) in all territories worldwide, (ii) for the maximum 62 | duration provided by applicable law or treaty (including future time 63 | extensions), (iii) in any current or future medium and for any number of 64 | copies, and (iv) for any purpose whatsoever, including without limitation 65 | commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 66 | the Waiver for the benefit of each member of the public at large and to the 67 | detriment of Affirmer's heirs and successors, fully intending that such Waiver 68 | shall not be subject to revocation, rescission, cancellation, termination, or 69 | any other legal or equitable action to disrupt the quiet enjoyment of the Work 70 | by the public as contemplated by Affirmer's express Statement of Purpose. 71 | 72 | 3. Public License Fallback. Should any part of the Waiver for any reason be 73 | judged legally invalid or ineffective under applicable law, then the Waiver 74 | shall be preserved to the maximum extent permitted taking into account 75 | Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 76 | is so judged Affirmer hereby grants to each affected person a royalty-free, 77 | non transferable, non sublicensable, non exclusive, irrevocable and 78 | unconditional license to exercise Affirmer's Copyright and Related Rights in 79 | the Work (i) in all territories worldwide, (ii) for the maximum duration 80 | provided by applicable law or treaty (including future time extensions), (iii) 81 | in any current or future medium and for any number of copies, and (iv) for any 82 | purpose whatsoever, including without limitation commercial, advertising or 83 | promotional purposes (the "License"). The License shall be deemed effective as 84 | of the date CC0 was applied by Affirmer to the Work. Should any part of the 85 | License for any reason be judged legally invalid or ineffective under 86 | applicable law, such partial invalidity or ineffectiveness shall not 87 | invalidate the remainder of the License, and in such case Affirmer hereby 88 | affirms that he or she will not (i) exercise any of his or her remaining 89 | Copyright and Related Rights in the Work or (ii) assert any associated claims 90 | and causes of action with respect to the Work, in either case contrary to 91 | Affirmer's express Statement of Purpose. 92 | 93 | 4. Limitations and Disclaimers. 94 | 95 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 96 | surrendered, licensed or otherwise affected by this document. 97 | 98 | b. Affirmer offers the Work as-is and makes no representations or warranties 99 | of any kind concerning the Work, express, implied, statutory or otherwise, 100 | including without limitation warranties of title, merchantability, fitness 101 | for a particular purpose, non infringement, or the absence of latent or 102 | other defects, accuracy, or the present or absence of errors, whether or not 103 | discoverable, all to the greatest extent permissible under applicable law. 104 | 105 | c. Affirmer disclaims responsibility for clearing rights of other persons 106 | that may apply to the Work or any use thereof, including without limitation 107 | any person's Copyright and Related Rights in the Work. Further, Affirmer 108 | disclaims responsibility for obtaining any necessary consents, permissions 109 | or other rights required for any use of the Work. 110 | 111 | d. Affirmer understands and acknowledges that Creative Commons is not a 112 | party to this document and has no duty or obligation with respect to this 113 | CC0 or use of the Work. 114 | 115 | For more information, please see 116 | 117 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Trustworthy Mail # 2 | 3 | [![Latest Version](https://img.shields.io/pypi/v/trustymail.svg)](https://pypi.org/project/trustymail/) 4 | [![GitHub Build Status](https://github.com/cisagov/trustymail/workflows/build/badge.svg)](https://github.com/cisagov/trustymail/actions) 5 | [![CodeQL](https://github.com/cisagov/trustymail/workflows/CodeQL/badge.svg)](https://github.com/cisagov/trustymail/actions/workflows/codeql-analysis.yml) 6 | [![Coverage Status](https://coveralls.io/repos/github/cisagov/trustymail/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/trustymail?branch=develop) 7 | [![Known Vulnerabilities](https://snyk.io/test/github/cisagov/trustymail/develop/badge.svg)](https://snyk.io/test/github/cisagov/trustymail) 8 | 9 | `trustymail` is a tool that evaluates SPF/DMARC records set in a 10 | domain's DNS. It also checks the mail servers listed in a domain's MX 11 | records for STARTTLS support. It saves its results to CSV or JSON. 12 | 13 | ## Getting started ## 14 | 15 | `trustymail` requires **Python 3.6+**. Python 2 is not supported. 16 | 17 | ### Local installation ### 18 | 19 | `trustymail` can be installed directly via pip: 20 | 21 | ```console 22 | pip install trustymail 23 | ``` 24 | 25 | It can then be run directly: 26 | 27 | ```console 28 | trustymail [options] example.com 29 | ``` 30 | 31 | or 32 | 33 | ```console 34 | python3 -m trustymail [options] example.com 35 | ``` 36 | 37 | ### Using Docker (optional) ### 38 | 39 | ```console 40 | ./run [opts] 41 | ``` 42 | 43 | `opts` are the same arguments that would get passed to `trustymail`. 44 | 45 | ### Usage and examples ### 46 | 47 | ```console 48 | trustymail [options] INPUT 49 | 50 | trustymail dhs.gov 51 | trustymail --output=homeland.csv --debug cisa.gov dhs.gov us-cert.gov usss.gov 52 | trustymail agencies.csv 53 | ``` 54 | 55 | Note: if INPUT ends with `.csv`, domains will be read from CSV. CSV 56 | output will always be written to disk, defaulting to `results.csv`. 57 | 58 | #### Options #### 59 | 60 | ```console 61 | -h --help Show this message. 62 | -o --output=OUTFILE Name of output file. (Default results) 63 | -t --timeout=TIMEOUT The DNS lookup timeout in seconds. (Default is 5.) 64 | --smtp-timeout=TIMEOUT The SMTP connection timeout in seconds. (Default is 5.) 65 | --smtp-localhost=HOSTNAME The hostname to use when connecting to SMTP 66 | servers. (Default is the FQDN of the host from 67 | which trustymail is being run.) 68 | --smtp-ports=PORTS A comma-delimited list of ports at which to look 69 | for SMTP servers. (Default is '25,465,587'.) 70 | --no-smtp-cache Do not cache SMTP results during the run. This 71 | may results in slower scans due to testing the 72 | same mail servers multiple times. 73 | --mx Only check MX records. 74 | --starttls Only check MX records and STARTTLS support. 75 | (Implies --mx.) 76 | --spf Only check SPF records. 77 | --dmarc Only check DMARC records. 78 | --json Output is in JSON format. (Default is CSV.) 79 | --debug Output should include more verbose logging. 80 | --dns=HOSTNAMES A comma-delimited list of DNS servers to query 81 | against. For example, if you want to use 82 | Google's DNS then you would use the 83 | value --dns-hostnames='8.8.8.8,8.8.4.4'. By 84 | default the DNS configuration of the host OS 85 | (/etc/resolv.conf) is used. Note that 86 | the host's DNS configuration is not used at all 87 | if this option is used. 88 | --psl-filename=FILENAME The name of the file where the public suffix list 89 | (PSL) cache will be saved. If set to the name of 90 | an existing file then that file will be used as 91 | the PSL. If not present then the PSL cache will 92 | be saved to a file in the current directory called 93 | public_suffix_list.dat. 94 | --psl-read-only If present, then the public suffix list (PSL) 95 | cache will be read but never overwritten. This 96 | is useful when running in AWS Lambda, for 97 | instance, where the local filesystem is read-only. 98 | ``` 99 | 100 | ## What's checked? ## 101 | 102 | For a given domain, MX records, SPF records (TXT), DMARC (TXT, at 103 | `_dmarc.`), and support for STARTTLS are checked. Resource records can 104 | also be checked for DNSSEC if the resolver used is DNSSEC-aware. 105 | 106 | The following values are returned in `results.csv`: 107 | 108 | ### Domain and redirect info ### 109 | 110 | - `Domain` - The domain you're scanning! 111 | - `Base Domain` - The base domain of `Domain`. For example, for a 112 | Domain of `sub.example.gov`, the Base Domain will be 113 | `example.gov`. Usually this is the second-level domain, but 114 | `trustymail` will download and factor in the [Public Suffix 115 | List](https://publicsuffix.org) when calculating the base domain. 116 | - `Live` - The domain is actually published in the DNS. 117 | 118 | ### Mail sending ### 119 | 120 | - `MX Record` - If an MX record was found that contains at least a 121 | single mail server. 122 | - `MX Record DNSSEC` - A boolean value indicating whether or not the 123 | DNS record is protected by DNSSEC. 124 | - `Mail Servers` - The list of hosts found in the MX record. 125 | - `Mail Server Ports Tested` - A list of the ports tested for SMTP and 126 | STARTTLS support. 127 | - `Domain Supports SMTP` - True if and only if **any** mail servers 128 | specified in a MX record associated with the domain supports SMTP. 129 | - `Domain Supports SMTP Results` - A list of the mail server and port 130 | combinations that support SMTP. 131 | - `Domain Supports STARTTLS` - True if and only if **all** mail 132 | servers that support SMTP also support STARTTLS. 133 | - `Domain Supports STARTTLS Results` - A list of the mail server and 134 | port combinations that support STARTTLS. 135 | 136 | ### Sender Policy Framework (SPF) ### 137 | 138 | - `SPF Record` - Whether or not a SPF record was found. 139 | - `SPF Record DNSSEC` - A boolean value indicating whether or not the 140 | DNS record is protected by DNSSEC. 141 | - `Valid SPF` - Whether the SPF record found is syntactically correct, 142 | per RFC 4408. 143 | - `SPF Results` - The textual representation of any SPF record found 144 | for the domain. 145 | 146 | ### Domain-based Message Authentication, Reporting, and Conformance (DMARC) ### 147 | 148 | - `DMARC Record` - True/False whether or not a DMARC record was found. 149 | - `DMARC Record DNSSEC` - A boolean value indicating whether or not 150 | the DNS record is protected by DNSSEC. 151 | - `Valid DMARC` - Whether the DMARC record found is syntactically 152 | correct. 153 | - `DMARC Results` - The DMARC record that was discovered when querying 154 | DNS. 155 | - `DMARC Record on Base Domain`, `DMARC Record on Base Domain DNSSEC`, 156 | `Valid DMARC Record on Base Domain`, `DMARC Results on Base 157 | Domain` - Same definition as above, but returns the result for the 158 | Base Domain. This is important in DMARC because if there isn't a 159 | DMARC record at the domain, the base domain (or "Organizational 160 | Domain", per [RFC 161 | 7489](https://tools.ietf.org/html/rfc7489#section-6.6.3)), is 162 | checked and applied. 163 | - `DMARC Policy` - An adjudication, based on any policies found in 164 | `DMARC Results` and `DMARC Results on Base Domain`, of the relevant 165 | DMARC policy that applies. 166 | - `DMARC Subdomain Policy` - An adjudication, based on any policies 167 | found in `DMARC Results` and `DMARC Results on Base Domain`, of the 168 | relevant DMARC subdomain policy that applies. 169 | - `DMARC Policy Percentage` - The percentage of mail that should be 170 | subjected to the `DMARC Policy` according to the `DMARC Results`. 171 | - `DMARC Aggregate Report URIs` - A list of the DMARC aggregate report 172 | URIs specified by the domain. 173 | - `DMARC Forensic Report URIs` - A list of the DMARC forensic report 174 | URIs specified by the domain. 175 | - `DMARC Has Aggregate Report URI` - A boolean value that indicates if 176 | `DMARC Results` included `rua` URIs that tell recipients where to 177 | send DMARC aggregate reports. 178 | - `DMARC Has Forensic Report URI` - A boolean value that indicates if 179 | `DMARC Results` included `ruf` URIs that tell recipients where to 180 | send DMARC forensic reports. 181 | - `DMARC Reporting Address Acceptance Error` - A boolean value that is 182 | True if one or more of the domains listed in the aggregate and 183 | forensic report URIs does not indicate that it accepts DMARC reports 184 | from the domain being tested. 185 | 186 | ### Everything else ### 187 | 188 | - `Syntax Errors` - A list of syntax errors that were encountered when 189 | analyzing SPF records. 190 | - `Debug Info` - A list of any other warnings or errors encountered, 191 | such as DNS failures. These can be helpful when determining how 192 | `trustymail` reached its conclusions, and are indispensible for bug 193 | reports. 194 | 195 | ## Contributing ## 196 | 197 | We welcome contributions! Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for 198 | details. 199 | 200 | ## License ## 201 | 202 | This project is in the worldwide [public domain](LICENSE). 203 | -------------------------------------------------------------------------------- /bump-version: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # bump-version [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) 4 | # bump-version --list-files 5 | 6 | set -o nounset 7 | set -o errexit 8 | set -o pipefail 9 | 10 | # Stores the canonical version for the project. 11 | VERSION_FILE=src/trustymail/_version.py 12 | # Files that should be updated with the new version. 13 | VERSION_FILES=("$VERSION_FILE") 14 | 15 | USAGE=$( 16 | cat << END_OF_LINE 17 | Update the version of the project. 18 | 19 | Usage: 20 | ${0##*/} [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) 21 | ${0##*/} --list-files 22 | ${0##*/} (-h | --help) 23 | 24 | Options: 25 | -h | --help Show this message. 26 | --push Perform a \`git push\` after updating the version. 27 | --label LABEL Specify the label to use when updating the build or prerelease version. 28 | --list-files List the files that will be updated when the version is bumped. 29 | END_OF_LINE 30 | ) 31 | 32 | old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) 33 | # Comment out periods so they are interpreted as periods and don't 34 | # just match any character 35 | old_version_regex=${old_version//\./\\\.} 36 | new_version="$old_version" 37 | 38 | bump_part="" 39 | label="" 40 | commit_prefix="Bump" 41 | with_push=false 42 | commands_with_label=("build" "prerelease") 43 | commands_with_prerelease=("major" "minor" "patch") 44 | with_prerelease=false 45 | 46 | ####################################### 47 | # Display an error message, the help information, and exit with a non-zero status. 48 | # Arguments: 49 | # Error message. 50 | ####################################### 51 | function invalid_option() { 52 | echo "$1" 53 | echo "$USAGE" 54 | exit 1 55 | } 56 | 57 | ####################################### 58 | # Bump the version using the provided command. 59 | # Arguments: 60 | # The version to bump. 61 | # The command to bump the version. 62 | # Returns: 63 | # The new version. 64 | ####################################### 65 | function bump_version() { 66 | local temp_version 67 | temp_version=$(python -c "import semver; print(semver.parse_version_info('$1').${2})") 68 | echo "$temp_version" 69 | } 70 | 71 | if [ $# -eq 0 ]; then 72 | echo "$USAGE" 73 | exit 1 74 | else 75 | while [ $# -gt 0 ]; do 76 | case $1 in 77 | --push) 78 | if [ "$with_push" = true ]; then 79 | invalid_option "Push has already been set." 80 | fi 81 | 82 | with_push=true 83 | shift 84 | ;; 85 | --label) 86 | if [ -n "$label" ]; then 87 | invalid_option "Label has already been set." 88 | fi 89 | 90 | label="$2" 91 | shift 2 92 | ;; 93 | build | finalize | major | minor | patch) 94 | if [ -n "$bump_part" ]; then 95 | invalid_option "Only one version part should be bumped at a time." 96 | fi 97 | 98 | bump_part="$1" 99 | shift 100 | ;; 101 | prerelease) 102 | with_prerelease=true 103 | shift 104 | ;; 105 | show) 106 | echo "$old_version" 107 | exit 0 108 | ;; 109 | -h | --help) 110 | echo "$USAGE" 111 | exit 0 112 | ;; 113 | --list-files) 114 | printf '%s\n' "${VERSION_FILES[@]}" 115 | exit 0 116 | ;; 117 | *) 118 | invalid_option "Invalid option: $1" 119 | ;; 120 | esac 121 | done 122 | fi 123 | 124 | if [ -n "$label" ] && [ "$with_prerelease" = false ] && [[ ! " ${commands_with_label[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then 125 | invalid_option "Setting the label is only allowed for the following commands: ${commands_with_label[*]}" 126 | fi 127 | 128 | if [ "$with_prerelease" = true ] && [ -n "$bump_part" ] && [[ ! " ${commands_with_prerelease[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then 129 | invalid_option "Changing the prerelease is only allowed in conjunction with the following commands: ${commands_with_prerelease[*]}" 130 | fi 131 | 132 | label_option="" 133 | if [ -n "$label" ]; then 134 | label_option="token='$label'" 135 | fi 136 | 137 | if [ -n "$bump_part" ]; then 138 | if [ "$bump_part" = "finalize" ]; then 139 | commit_prefix="Finalize" 140 | bump_command="finalize_version()" 141 | elif [ "$bump_part" = "build" ]; then 142 | bump_command="bump_${bump_part}($label_option)" 143 | else 144 | bump_command="bump_${bump_part}()" 145 | fi 146 | new_version=$(bump_version "$old_version" "$bump_command") 147 | echo Changing version from "$old_version" to "$new_version" 148 | fi 149 | 150 | if [ "$with_prerelease" = true ]; then 151 | bump_command="bump_prerelease($label_option)" 152 | temp_version=$(bump_version "$new_version" "$bump_command") 153 | echo Changing version from "$new_version" to "$temp_version" 154 | new_version="$temp_version" 155 | fi 156 | 157 | tmp_file=/tmp/version.$$ 158 | for version_file in "${VERSION_FILES[@]}"; do 159 | if [ ! -f "$version_file" ]; then 160 | echo Missing expected file: "$version_file" 161 | exit 1 162 | fi 163 | sed "s/$old_version_regex/$new_version/" "$version_file" > $tmp_file 164 | mv $tmp_file "$version_file" 165 | done 166 | 167 | git add "${VERSION_FILES[@]}" 168 | git commit --message "$commit_prefix version from $old_version to $new_version" 169 | 170 | if [ "$with_push" = true ]; then 171 | git push 172 | fi 173 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | # Increase verbosity, display extra test summary info for tests that did not pass, 3 | # display code coverage results, and enable debug logging 4 | addopts = --verbose -ra --cov --log-cli-level=DEBUG 5 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | --editable .[dev] 2 | --requirement requirements-test.txt 3 | ipython 4 | mypy 5 | # The bump-version script requires at least version 3 of semver. 6 | semver>=3 7 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | --editable .[test] 2 | --requirement requirements.txt 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Note: Add any additional requirements to setup.py's install_requires field 2 | --editable . 3 | wheel 4 | -------------------------------------------------------------------------------- /run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | 6 | docker build -t trustymail/cli . 7 | 8 | docker run --rm -it \ 9 | --name trustymail \ 10 | --volume "$(pwd)":/workspace \ 11 | --workdir="/workspace" \ 12 | --user "$(id -u)" \ 13 | trustymail/cli "$@" 14 | -------------------------------------------------------------------------------- /setup-env: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o nounset 4 | set -o errexit 5 | set -o pipefail 6 | 7 | USAGE=$( 8 | cat << 'END_OF_LINE' 9 | Configure a development environment for this repository. 10 | 11 | It does the following: 12 | - Allows the user to specify the Python version to use for the virtual environment. 13 | - Allows the user to specify a name for the virtual environment. 14 | - Verifies pyenv and pyenv-virtualenv are installed. 15 | - Creates the Python virtual environment. 16 | - Configures the activation of the virtual enviroment for the repo directory. 17 | - Installs the requirements needed for development (including mypy type stubs). 18 | - Installs git pre-commit hooks. 19 | - Configures git remotes for upstream "lineage" repositories. 20 | 21 | Usage: 22 | setup-env [--venv-name venv_name] [--python-version python_version] 23 | setup-env (-h | --help) 24 | 25 | Options: 26 | -f | --force Delete virtual enviroment if it already exists. 27 | -h | --help Show this message. 28 | -i | --install-hooks Install hook environments for all environments in the 29 | pre-commit config file. 30 | -l | --list-versions List available Python versions and select one interactively. 31 | -v | --venv-name Specify the name of the virtual environment. 32 | -p | --python-version Specify the Python version for the virtual environment. 33 | 34 | END_OF_LINE 35 | ) 36 | 37 | # Display pyenv's installed Python versions 38 | python_versions() { 39 | pyenv versions --bare --skip-aliases --skip-envs 40 | } 41 | 42 | check_python_version() { 43 | local version=$1 44 | 45 | # This is a valid regex for semantically correct Python version strings. 46 | # For more information see here: 47 | # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string 48 | # Break down the regex into readable parts major.minor.patch 49 | local major="0|[1-9]\d*" 50 | local minor="0|[1-9]\d*" 51 | local patch="0|[1-9]\d*" 52 | 53 | # Splitting the prerelease part for readability 54 | # Start of the prerelease 55 | local prerelease="(?:-" 56 | # Numeric or alphanumeric identifiers 57 | local prerelease+="(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)" 58 | # Additional dot-separated identifiers 59 | local prerelease+="(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*" 60 | # End of the prerelease, making it optional 61 | local prerelease+=")?" 62 | # Optional build metadata 63 | local build="(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?" 64 | 65 | # Final regex composed of parts 66 | local regex="^($major)\.($minor)\.($patch)$prerelease$build$" 67 | 68 | # This checks if the Python version does not match the regex pattern specified in $regex, 69 | # using Perl for regex matching. If the pattern is not found, then prompt the user with 70 | # the invalid version message. 71 | if ! echo "$version" | perl -ne "exit(!/$regex/)"; then 72 | echo "Invalid version of Python: Python follows semantic versioning," \ 73 | "so any version string that is not a valid semantic version is an" \ 74 | "invalid version of Python." 75 | exit 1 76 | # Else if the Python version isn't installed then notify the user. 77 | # grep -E is used for searching through text lines that match the 78 | # specific version. 79 | elif ! python_versions | grep -E "^${version}$" > /dev/null; then 80 | echo "Error: Python version $version is not installed." 81 | echo "Installed Python versions are:" 82 | python_versions 83 | exit 1 84 | else 85 | echo "Using Python version $version" 86 | fi 87 | } 88 | 89 | # Flag to force deletion and creation of virtual environment 90 | FORCE=0 91 | 92 | # Initialize the other flags 93 | INSTALL_HOOKS=0 94 | LIST_VERSIONS=0 95 | PYTHON_VERSION="" 96 | VENV_NAME="" 97 | 98 | # Define long options 99 | LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" 100 | 101 | # Define short options for getopt 102 | SHORTOPTS="fhilp:v:" 103 | 104 | # Check for GNU getopt by matching a specific pattern ("getopt from util-linux") 105 | # in its version output. This approach presumes the output format remains stable. 106 | # Be aware that format changes could invalidate this check. 107 | if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then 108 | cat << 'END_OF_LINE' 109 | 110 | Please note, this script requires GNU getopt due to its enhanced 111 | functionality and compatibility with certain script features that 112 | are not supported by the POSIX getopt found in some systems, particularly 113 | those with a non-GNU version of getopt. This distinction is crucial 114 | as a system might have a non-GNU version of getopt installed by default, 115 | which could lead to unexpected behavior. 116 | 117 | On macOS, we recommend installing brew (https://brew.sh/). Then installation 118 | is as simple as `brew install gnu-getopt` and adding this to your 119 | profile: 120 | 121 | export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" 122 | 123 | GNU getopt must be explicitly added to the PATH since it 124 | is keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean). 125 | 126 | END_OF_LINE 127 | exit 1 128 | fi 129 | 130 | # Check to see if pyenv is installed 131 | if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then 132 | echo "pyenv and pyenv-virtualenv are required." 133 | if [[ "$OSTYPE" == "darwin"* ]]; then 134 | cat << 'END_OF_LINE' 135 | 136 | On macOS, we recommend installing brew, https://brew.sh/. Then installation 137 | is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your 138 | profile: 139 | 140 | eval "$(pyenv init -)" 141 | eval "$(pyenv virtualenv-init -)" 142 | 143 | END_OF_LINE 144 | 145 | fi 146 | cat << 'END_OF_LINE' 147 | For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want 148 | to use "brew") you can use https://github.com/pyenv/pyenv-installer to install 149 | the necessary tools. Before running this ensure that you have installed the 150 | prerequisites for your platform according to the pyenv wiki page, 151 | https://github.com/pyenv/pyenv/wiki/common-build-problems. 152 | 153 | On WSL you should treat your platform as whatever Linux distribution you've 154 | chosen to install. 155 | 156 | Once you have installed "pyenv" you will need to add the following lines to 157 | your ".bashrc": 158 | 159 | export PATH="$PATH:$HOME/.pyenv/bin" 160 | eval "$(pyenv init -)" 161 | eval "$(pyenv virtualenv-init -)" 162 | END_OF_LINE 163 | exit 1 164 | fi 165 | 166 | # Use GNU getopt to parse options 167 | if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then 168 | echo "Error parsing options" 169 | exit 1 170 | fi 171 | eval set -- "$PARSED" 172 | 173 | while true; do 174 | case "$1" in 175 | -f | --force) 176 | FORCE=1 177 | shift 178 | ;; 179 | -h | --help) 180 | echo "$USAGE" 181 | exit 0 182 | ;; 183 | -i | --install-hooks) 184 | INSTALL_HOOKS=1 185 | shift 186 | ;; 187 | -l | --list-versions) 188 | LIST_VERSIONS=1 189 | shift 190 | ;; 191 | -p | --python-version) 192 | PYTHON_VERSION="$2" 193 | shift 2 194 | # Check the Python version being passed in. 195 | check_python_version "$PYTHON_VERSION" 196 | ;; 197 | -v | --venv-name) 198 | VENV_NAME="$2" 199 | shift 2 200 | ;; 201 | --) 202 | shift 203 | break 204 | ;; 205 | *) 206 | # Unreachable due to GNU getopt handling all options 207 | echo "Programming error" 208 | exit 64 209 | ;; 210 | esac 211 | done 212 | 213 | # Determine the virtual environment name 214 | if [ -n "$VENV_NAME" ]; then 215 | # Use the user-provided environment name 216 | env_name="$VENV_NAME" 217 | else 218 | # Set the environment name to the last part of the working directory. 219 | env_name=${PWD##*/} 220 | fi 221 | 222 | # List Python versions and select one interactively. 223 | if [ $LIST_VERSIONS -ne 0 ]; then 224 | echo Available Python versions: 225 | python_versions 226 | # Read the user's desired Python version. 227 | # -r: treat backslashes as literal, -p: display prompt before input. 228 | read -r -p "Enter the desired Python version: " PYTHON_VERSION 229 | # Check the Python version being passed in. 230 | check_python_version "$PYTHON_VERSION" 231 | fi 232 | 233 | # Remove any lingering local configuration. 234 | if [ $FORCE -ne 0 ]; then 235 | rm -f .python-version 236 | pyenv virtualenv-delete --force "${env_name}" || true 237 | elif [[ -f .python-version ]]; then 238 | cat << 'END_OF_LINE' 239 | An existing .python-version file was found. Either remove this file yourself 240 | or re-run with the --force option to have it deleted along with the associated 241 | virtual environment. 242 | 243 | rm .python-version 244 | 245 | END_OF_LINE 246 | exit 1 247 | fi 248 | 249 | # Create a new virtual environment for this project 250 | # 251 | # If $PYTHON_VERSION is undefined then the current pyenv Python version will be used. 252 | # 253 | # We can't quote ${PYTHON_VERSION:=} below since if the variable is 254 | # undefined then we want nothing to appear; this is the reason for the 255 | # "shellcheck disable" line below. 256 | # 257 | # shellcheck disable=SC2086 258 | if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then 259 | cat << END_OF_LINE 260 | An existing virtual environment named $env_name was found. Either delete this 261 | environment yourself or re-run with the --force option to have it deleted. 262 | 263 | pyenv virtualenv-delete ${env_name} 264 | 265 | END_OF_LINE 266 | exit 1 267 | fi 268 | 269 | # Set the local application-specific Python version(s) by writing the 270 | # version name to a file named `.python-version'. 271 | pyenv local "${env_name}" 272 | 273 | # Upgrade pip and friends 274 | python3 -m pip install --upgrade pip setuptools wheel 275 | 276 | # Find a requirements file (if possible) and install 277 | for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do 278 | if [[ -f $req_file ]]; then 279 | pip install --requirement $req_file 280 | break 281 | fi 282 | done 283 | 284 | # Install git pre-commit hooks now or later. 285 | pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} 286 | 287 | # Setup git remotes from lineage configuration 288 | # This could fail if the remotes are already setup, but that is ok. 289 | set +o errexit 290 | 291 | eval "$( 292 | python3 << 'END_OF_LINE' 293 | from pathlib import Path 294 | import yaml 295 | import sys 296 | 297 | LINEAGE_CONFIG = Path(".github/lineage.yml") 298 | 299 | if not LINEAGE_CONFIG.exists(): 300 | print("No lineage configuration found.", file=sys.stderr) 301 | sys.exit(0) 302 | 303 | with LINEAGE_CONFIG.open("r") as f: 304 | lineage = yaml.safe_load(stream=f) 305 | 306 | if lineage["version"] == "1": 307 | for parent_name, v in lineage["lineage"].items(): 308 | remote_url = v["remote-url"] 309 | print(f"git remote add {parent_name} {remote_url};") 310 | print(f"git remote set-url --push {parent_name} no_push;") 311 | else: 312 | print(f'Unsupported lineage version: {lineage["version"]}', file=sys.stderr) 313 | END_OF_LINE 314 | )" 315 | 316 | # Install all necessary mypy type stubs 317 | mypy --install-types --non-interactive src/ 318 | 319 | # Qapla' 320 | echo "Success!" 321 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is the setup module for the trustymail project. 3 | 4 | Based on: 5 | 6 | - https://packaging.python.org/distributing/ 7 | - https://github.com/pypa/sampleproject/blob/master/setup.py 8 | - https://blog.ionelmc.ro/2014/05/25/python-packaging/#the-structure 9 | """ 10 | 11 | # Standard Python Libraries 12 | import codecs 13 | from glob import glob 14 | from os.path import abspath, basename, dirname, join, splitext 15 | 16 | # Third-Party Libraries 17 | from setuptools import find_packages, setup 18 | 19 | 20 | def readme(): 21 | """Read in and return the contents of the project's README.md file.""" 22 | with open("README.md", encoding="utf-8") as f: 23 | return f.read() 24 | 25 | 26 | # Below two methods were pulled from: 27 | # https://packaging.python.org/guides/single-sourcing-package-version/ 28 | def read(rel_path): 29 | """Open a file for reading from a given relative path.""" 30 | here = abspath(dirname(__file__)) 31 | with codecs.open(join(here, rel_path), "r") as fp: 32 | return fp.read() 33 | 34 | 35 | def get_version(version_file): 36 | """Extract a version number from the given file path.""" 37 | for line in read(version_file).splitlines(): 38 | if line.startswith("__version__"): 39 | delim = '"' if '"' in line else "'" 40 | return line.split(delim)[1] 41 | raise RuntimeError("Unable to find version string.") 42 | 43 | 44 | setup( 45 | name="trustymail", 46 | # Versions should comply with PEP440 47 | version=get_version("src/trustymail/_version.py"), 48 | description="Scan domains and return data based on trustworthy email best practices", 49 | long_description=readme(), 50 | long_description_content_type="text/markdown", 51 | # Landing page for CISA's cybersecurity mission 52 | url="https://www.cisa.gov/cybersecurity", 53 | # Additional URLs for this project per 54 | # https://packaging.python.org/guides/distributing-packages-using-setuptools/#project-urls 55 | project_urls={ 56 | "Source": "https://github.com/cisagov/trustymail", 57 | "Tracker": "https://github.com/cisagov/trustymail/issues", 58 | }, 59 | # Author details 60 | author="Cybersecurity and Infrastructure Security Agency", 61 | author_email="github@cisa.dhs.gov", 62 | license="License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", 63 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 64 | classifiers=[ 65 | # How mature is this project? Common values are 66 | # 3 - Alpha 67 | # 4 - Beta 68 | # 5 - Production/Stable 69 | "Development Status :: 4 - Beta", 70 | # Indicate who your project is intended for 71 | "Intended Audience :: Developers", 72 | # Pick your license as you wish (should match "license" above) 73 | "License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", 74 | # Specify the Python versions you support here. In particular, ensure 75 | # that you indicate whether you support Python 2, Python 3 or both. 76 | "Programming Language :: Python :: 3", 77 | "Programming Language :: Python :: 3 :: Only", 78 | "Programming Language :: Python :: 3.7", 79 | "Programming Language :: Python :: 3.8", 80 | "Programming Language :: Python :: 3.9", 81 | "Programming Language :: Python :: 3.10", 82 | "Programming Language :: Python :: 3.11", 83 | "Programming Language :: Python :: 3.12", 84 | "Programming Language :: Python :: 3.13", 85 | "Programming Language :: Python :: Implementation :: CPython", 86 | ], 87 | python_requires=">=3.7", 88 | # What does your project relate to? 89 | keywords="email authentication STARTTLS", 90 | packages=find_packages(where="src"), 91 | package_dir={"": "src"}, 92 | py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], 93 | include_package_data=True, 94 | install_requires=[ 95 | "dnspython", 96 | "docopt", 97 | "publicsuffixlist[update]", 98 | "py3dns", 99 | "pyspf", 100 | "requests", 101 | "setuptools", 102 | ], 103 | extras_require={ 104 | # IMPORTANT: Keep type hinting-related dependencies of the dev section 105 | # in sync with the mypy pre-commit hook configuration (see 106 | # .pre-commit-config.yaml). Any changes to type hinting-related 107 | # dependencies here should be reflected in the additional_dependencies 108 | # field of the mypy pre-commit hook to avoid discrepancies in type 109 | # checking between environments. 110 | "dev": [ 111 | "types-docopt", 112 | "types-requests", 113 | "types-setuptools", 114 | ], 115 | "test": [ 116 | "coverage", 117 | "coveralls", 118 | "pre-commit", 119 | "pytest-cov", 120 | "pytest", 121 | ], 122 | }, 123 | # Conveniently allows one to run the CLI tool as `trustymail` 124 | entry_points={"console_scripts": ["trustymail = trustymail.cli:main"]}, 125 | ) 126 | -------------------------------------------------------------------------------- /src/trustymail/__init__.py: -------------------------------------------------------------------------------- 1 | """The trustymail library.""" 2 | 3 | # Standard Python Libraries 4 | from typing import List 5 | 6 | # We disable a Flake8 check for "Module imported but unused (F401)" here because 7 | # although this import is not directly used, it populates the value 8 | # package_name.__version__, which is used to get version information about this 9 | # Python package. 10 | from ._version import __version__ # noqa: F401 11 | 12 | __all__: List[str] = [] 13 | 14 | PublicSuffixListFilename = "public_suffix_list.dat" 15 | PublicSuffixListReadOnly = False 16 | -------------------------------------------------------------------------------- /src/trustymail/__main__.py: -------------------------------------------------------------------------------- 1 | """Code to run if this package is used as a Python module.""" 2 | 3 | from .cli import main 4 | 5 | main() 6 | -------------------------------------------------------------------------------- /src/trustymail/_version.py: -------------------------------------------------------------------------------- 1 | """This file defines the version of this module.""" 2 | 3 | __version__ = "1.1.0" 4 | -------------------------------------------------------------------------------- /src/trustymail/cli.py: -------------------------------------------------------------------------------- 1 | """trustymail: A tool for scanning DNS mail records for evaluating security. 2 | 3 | Usage: 4 | trustymail (INPUT ...) [options] 5 | trustymail (INPUT ...) [--output=OUTFILE] [--timeout=TIMEOUT] [--smtp-timeout=TIMEOUT] [--smtp-localhost=HOSTNAME] [--smtp-ports=PORTS] [--no-smtp-cache] [--mx] [--starttls] [--spf] [--dmarc] [--debug] [--json] [--dns=HOSTNAMES] [--psl-filename=FILENAME] [--psl-read-only] 6 | trustymail (-h | --help) 7 | 8 | Options: 9 | -h --help Show this message. 10 | -o --output=OUTFILE Name of output file. (Default results) 11 | -t --timeout=TIMEOUT The DNS lookup timeout in seconds. (Default is 5.) 12 | --smtp-timeout=TIMEOUT The SMTP connection timeout in seconds. (Default is 5.) 13 | --smtp-localhost=HOSTNAME The hostname to use when connecting to SMTP 14 | servers. (Default is the FQDN of the host from 15 | which trustymail is being run.) 16 | --smtp-ports=PORTS A comma-delimited list of ports at which to look 17 | for SMTP servers. (Default is '25,465,587'.) 18 | --no-smtp-cache Do not cache SMTP results during the run. This 19 | may results in slower scans due to testing the 20 | same mail servers multiple times. 21 | --mx Only check MX records. 22 | --starttls Only check MX records and STARTTLS support. 23 | (Implies --mx.) 24 | --spf Only check SPF records. 25 | --dmarc Only check DMARC records. 26 | --json Output is in JSON format. (Default is CSV.) 27 | --debug Output should include more verbose logging. 28 | --dns=HOSTNAMES A comma-delimited list of DNS servers to query 29 | against. For example, if you want to use 30 | Google's DNS then you would use the 31 | value --dns-hostnames='8.8.8.8,8.8.4.4'. By 32 | default the DNS configuration of the host OS 33 | (/etc/resolv.conf) is used. Note that 34 | the host's DNS configuration is not used at all 35 | if this option is used. 36 | --psl-filename=FILENAME The name of the file where the public suffix list 37 | (PSL) cache will be saved. If set to the name of 38 | an existing file then that file will be used as 39 | the PSL. If not present then the PSL cache will 40 | be saved to a file in the current directory called 41 | public_suffix_list.dat. 42 | --psl-read-only If present, then the public suffix list (PSL) 43 | cache will be read but never overwritten. This 44 | is useful when running in AWS Lambda, for 45 | instance, where the local filesystem is read-only. 46 | 47 | Notes: 48 | If no scan type options are specified, all are run against a given domain/input. 49 | """ 50 | 51 | # Standard Python Libraries 52 | # Built-in imports 53 | import errno 54 | import logging 55 | import os 56 | 57 | # Third-Party Libraries 58 | # Dependency imports 59 | import docopt 60 | 61 | # Local imports 62 | from . import trustymail 63 | from ._version import __version__ 64 | 65 | # The default ports to be checked to see if an SMTP server is listening. 66 | _DEFAULT_SMTP_PORTS = {25, 465, 587} 67 | 68 | 69 | def main(): 70 | """Perform a trustymail scan using the provided options.""" 71 | args = docopt.docopt(__doc__, version=__version__) 72 | 73 | # Monkey patching trustymail to make it cache the PSL where we want 74 | if args["--psl-filename"] is not None: 75 | trustymail.PublicSuffixListFilename = args["--psl-filename"] 76 | # Monkey patching trustymail to make the PSL cache read-only 77 | if args["--psl-read-only"]: 78 | trustymail.PublicSuffixListReadOnly = True 79 | # cisagov Libraries 80 | import trustymail.trustymail as tmail 81 | 82 | log_level = logging.WARN 83 | if args["--debug"]: 84 | log_level = logging.DEBUG 85 | logging.basicConfig(format="%(asctime)-15s %(message)s", level=log_level) 86 | 87 | # Allow for user to input a csv for many domain names. 88 | if args["INPUT"][0].endswith(".csv"): 89 | domains = tmail.domain_list_from_csv(open(args["INPUT"][0])) 90 | else: 91 | domains = args["INPUT"] 92 | 93 | if args["--timeout"] is not None: 94 | timeout = int(args["--timeout"]) 95 | else: 96 | timeout = 5 97 | 98 | if args["--smtp-timeout"] is not None: 99 | smtp_timeout = int(args["--smtp-timeout"]) 100 | else: 101 | smtp_timeout = 5 102 | 103 | if args["--smtp-localhost"] is not None: 104 | smtp_localhost = args["--smtp-localhost"] 105 | else: 106 | smtp_localhost = None 107 | 108 | if args["--smtp-ports"] is not None: 109 | smtp_ports = {int(port) for port in args["--smtp-ports"].split(",")} 110 | else: 111 | smtp_ports = _DEFAULT_SMTP_PORTS 112 | 113 | if args["--dns"] is not None: 114 | dns_hostnames = args["--dns"].split(",") 115 | else: 116 | dns_hostnames = None 117 | 118 | # --starttls implies --mx 119 | if args["--starttls"]: 120 | args["--mx"] = True 121 | 122 | # User might not want every scan performed. 123 | scan_types = { 124 | "mx": args["--mx"], 125 | "starttls": args["--starttls"], 126 | "spf": args["--spf"], 127 | "dmarc": args["--dmarc"], 128 | } 129 | 130 | domain_scans = [] 131 | for domain_name in domains: 132 | domain_scans.append( 133 | tmail.scan( 134 | domain_name, 135 | timeout, 136 | smtp_timeout, 137 | smtp_localhost, 138 | smtp_ports, 139 | not args["--no-smtp-cache"], 140 | scan_types, 141 | dns_hostnames, 142 | ) 143 | ) 144 | 145 | # Default output file name is results. 146 | if args["--output"] is None: 147 | output_file_name = "results" 148 | else: 149 | output_file_name = args["--output"] 150 | 151 | # Ensure file extension is present in filename. 152 | if args["--json"] and ".json" not in output_file_name: 153 | output_file_name += ".json" 154 | elif ".csv" not in output_file_name: 155 | output_file_name += ".csv" 156 | 157 | if args["--json"]: 158 | json_out = tmail.generate_json(domain_scans) 159 | if args["--output"] is None: 160 | print(json_out) 161 | else: 162 | write(json_out, output_file_name) 163 | logging.warn("Wrote results to %s." % output_file_name) 164 | else: 165 | tmail.generate_csv(domain_scans, output_file_name) 166 | 167 | 168 | def write(content, out_file): 169 | """Write the provided content to a file after ensuring all intermediate directories exist.""" 170 | parent = os.path.dirname(out_file) 171 | if parent != "": 172 | mkdir_p(parent) 173 | 174 | f = open(out_file, "w") # no utf-8 in python 2 175 | f.write(content) 176 | f.close() 177 | 178 | 179 | # mkdir -p in python, from: 180 | # http://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python 181 | def mkdir_p(path): 182 | """Make a directory and all intermediate directories in its path.""" 183 | try: 184 | os.makedirs(path) 185 | except OSError as exc: # Python >2.5 186 | if exc.errno == errno.EEXIST: 187 | pass 188 | else: 189 | raise 190 | -------------------------------------------------------------------------------- /src/trustymail/domain.py: -------------------------------------------------------------------------------- 1 | """Provide a data model for domains and some utility functions.""" 2 | 3 | # Standard Python Libraries 4 | from collections import OrderedDict 5 | from datetime import datetime, timedelta 6 | from os import path, stat, utime 7 | from typing import Dict 8 | 9 | # Third-Party Libraries 10 | from publicsuffixlist.compat import PublicSuffixList 11 | from publicsuffixlist.update import updatePSL 12 | 13 | from . import PublicSuffixListFilename, PublicSuffixListReadOnly, trustymail 14 | 15 | 16 | def get_psl(): 17 | """Get the Public Suffix List - either new, or cached in the CWD for 24 hours. 18 | 19 | Returns 20 | ------- 21 | PublicSuffixList: An instance of PublicSuffixList loaded with a cached or updated list 22 | """ 23 | # Download the PSL if necessary 24 | if not PublicSuffixListReadOnly: 25 | if not path.exists(PublicSuffixListFilename): 26 | updatePSL(PublicSuffixListFilename) 27 | utime(PublicSuffixListFilename, None) # Set mtime to now 28 | else: 29 | psl_age = datetime.now() - datetime.fromtimestamp( 30 | stat(PublicSuffixListFilename).st_mtime 31 | ) 32 | if psl_age > timedelta(hours=24): 33 | updatePSL(PublicSuffixListFilename) 34 | utime(PublicSuffixListFilename, None) # Set mtime to now 35 | 36 | with open(PublicSuffixListFilename, encoding="utf-8") as psl_file: 37 | psl = PublicSuffixList(psl_file) 38 | 39 | return psl 40 | 41 | 42 | def get_public_suffix(domain): 43 | """Return the public suffix of a given domain.""" 44 | public_list = get_psl() 45 | 46 | return public_list.get_public_suffix(domain) 47 | 48 | 49 | def format_list(record_list): 50 | """Format a list into a string to increase readability in CSV.""" 51 | # record_list should only be a list, not an integer, None, or 52 | # anything else. Thus this if clause handles only empty 53 | # lists. This makes a "null" appear in the JSON output for 54 | # empty lists, as expected. 55 | if not record_list: 56 | return None 57 | 58 | return ", ".join(record_list) 59 | 60 | 61 | class Domain: 62 | """Store information about a domain.""" 63 | 64 | base_domains: Dict[str, "Domain"] = {} 65 | 66 | def __init__( 67 | self, 68 | domain_name, 69 | timeout, 70 | smtp_timeout, 71 | smtp_localhost, 72 | smtp_ports, 73 | smtp_cache, 74 | dns_hostnames, 75 | ): 76 | """Retrieve information about a given domain name.""" 77 | self.domain_name = domain_name.lower() 78 | 79 | self.base_domain_name = get_public_suffix(self.domain_name) 80 | 81 | self.is_base_domain = True 82 | self.base_domain = None 83 | if self.base_domain_name != self.domain_name: 84 | self.is_base_domain = False 85 | if self.base_domain_name not in Domain.base_domains: 86 | # Populate DMARC for parent. 87 | domain = trustymail.scan( 88 | self.base_domain_name, 89 | timeout, 90 | smtp_timeout, 91 | smtp_localhost, 92 | smtp_ports, 93 | smtp_cache, 94 | {"mx": False, "starttls": False, "spf": False, "dmarc": True}, 95 | dns_hostnames, 96 | ) 97 | Domain.base_domains[self.base_domain_name] = domain 98 | self.base_domain = Domain.base_domains[self.base_domain_name] 99 | 100 | # Start off assuming the host is live unless an error tells us otherwise. 101 | self.is_live = True 102 | 103 | # Keep entire record for potential future use. 104 | self.mx_records = None 105 | self.mx_records_dnssec = None 106 | self.spf = None 107 | self.spf_dnssec = None 108 | self.dmarc = None 109 | self.dmarc_dnssec = False 110 | self.dmarc_policy = None 111 | self.dmarc_subdomain_policy = None 112 | self.dmarc_pct = None 113 | self.dmarc_aggregate_uris = [] 114 | self.dmarc_forensic_uris = [] 115 | self.dmarc_has_aggregate_uri = False 116 | self.dmarc_has_forensic_uri = False 117 | self.dmarc_reports_address_error = False 118 | 119 | # Syntax validity - default spf to false as the lack of an SPF is a bad thing. 120 | self.valid_spf = False 121 | self.valid_dmarc = True 122 | self.syntax_errors = [] 123 | 124 | # Mail Info 125 | self.mail_servers = None 126 | 127 | # A dictionary for each port for each entry in mail_servers. 128 | # The dictionary's values indicate: 129 | # 1. Whether or not the server is listening on the port 130 | # 2. Whether or not the server supports SMTP 131 | # 3. Whether or not the server supports STARTTLS 132 | self.starttls_results = {} 133 | 134 | # A list of any debugging information collected while scanning records. 135 | self.debug_info = [] 136 | 137 | # A list of the ports tested for SMTP 138 | self.ports_tested = set() 139 | 140 | def has_mail(self): 141 | """Check if there are any mail servers associated with this domain.""" 142 | if self.mail_servers is not None: 143 | return len(self.mail_servers) > 0 144 | return None 145 | 146 | def has_supports_smtp(self): 147 | """Check if any of the mail servers associated with this domain are listening and support SMTP.""" 148 | result = None 149 | if len(self.starttls_results) > 0: 150 | result = ( 151 | len( 152 | filter( 153 | lambda x: self.starttls_results[x]["supports_smtp"], 154 | self.starttls_results.keys(), 155 | ) 156 | ) 157 | > 0 158 | ) 159 | return result 160 | 161 | def has_starttls(self): 162 | """Check if any of the mail servers associated with this domain are listening and support STARTTLS.""" 163 | result = None 164 | if len(self.starttls_results) > 0: 165 | result = ( 166 | len( 167 | filter( 168 | lambda x: self.starttls_results[x]["starttls"], 169 | self.starttls_results.keys(), 170 | ) 171 | ) 172 | > 0 173 | ) 174 | return result 175 | 176 | def has_spf(self): 177 | """Check if this domain has any Sender Policy Framework records.""" 178 | if self.spf is not None: 179 | return len(self.spf) > 0 180 | return None 181 | 182 | def has_dmarc(self): 183 | """Check if this domain has a Domain-based Message Authentication, Reporting, and Conformance record.""" 184 | if self.dmarc is not None: 185 | return len(self.dmarc) > 0 186 | return None 187 | 188 | def add_mx_record(self, record): 189 | """Add a mail server record for this domain.""" 190 | if self.mx_records is None: 191 | self.mx_records = [] 192 | self.mx_records.append(record) 193 | # The rstrip is because dnspython's string representation of 194 | # the record will contain a trailing period if it is a FQDN. 195 | if self.mail_servers is None: 196 | self.mail_servers = [] 197 | self.mail_servers.append(record.exchange.to_text().rstrip(".").lower()) 198 | 199 | def parent_has_dmarc(self): 200 | """Check if a domain or its parent has a Domain-based Message Authentication, Reporting, and Conformance record.""" 201 | ans = self.has_dmarc() 202 | if self.base_domain: 203 | ans = self.base_domain.has_dmarc() 204 | return ans 205 | 206 | def parent_dmarc_dnssec(self): 207 | """Get this domain or its parent's DMARC DNSSEC information.""" 208 | ans = self.dmarc_dnssec 209 | if self.base_domain: 210 | ans = self.base_domain.dmarc_dnssec 211 | return ans 212 | 213 | def parent_valid_dmarc(self): 214 | """Check if this domain or its parent have a valid DMARC record.""" 215 | ans = self.valid_dmarc 216 | if self.base_domain: 217 | return self.base_domain.valid_dmarc 218 | return ans 219 | 220 | def parent_dmarc_results(self): 221 | """Get this domain or its parent's DMARC information.""" 222 | ans = format_list(self.dmarc) 223 | if self.base_domain: 224 | ans = format_list(self.base_domain.dmarc) 225 | return ans 226 | 227 | def get_dmarc_policy(self): 228 | """Get this domain or its parent's DMARC policy.""" 229 | ans = self.dmarc_policy 230 | # If the policy was never set, or isn't in the list of valid 231 | # policies, check the parents. 232 | if ans is None or ans.lower() not in ["quarantine", "reject", "none"]: 233 | if self.base_domain: 234 | # We check the *subdomain* policy in case one was 235 | # explicitly set. If one was not explicitly set then 236 | # the subdomain policy is populated with the value for 237 | # the domain policy by trustymail.py anyway, in 238 | # accordance with the RFC 239 | # (https://tools.ietf.org/html/rfc7489#section-6.3). 240 | ans = self.base_domain.get_dmarc_subdomain_policy() 241 | else: 242 | ans = None 243 | return ans 244 | 245 | def get_dmarc_subdomain_policy(self): 246 | """Get this domain or its parent's DMARC subdomain policy.""" 247 | ans = self.dmarc_subdomain_policy 248 | # If the policy was never set, or isn't in the list of valid 249 | # policies, check the parents. 250 | if ans is None or ans.lower() not in ["quarantine", "reject", "none"]: 251 | if self.base_domain: 252 | ans = self.base_domain.get_dmarc_subdomain_policy() 253 | else: 254 | ans = None 255 | return ans 256 | 257 | def get_dmarc_pct(self): 258 | """Get this domain or its parent's DMARC percentage information.""" 259 | ans = self.dmarc_pct 260 | if not ans and self.base_domain: 261 | # Check the parents 262 | ans = self.base_domain.get_dmarc_pct() 263 | return ans 264 | 265 | def get_dmarc_has_aggregate_uri(self): 266 | """Get this domain or its parent's DMARC aggregate URI.""" 267 | ans = self.dmarc_has_aggregate_uri 268 | # If there are no aggregate URIs then check the parents. 269 | if not ans and self.base_domain: 270 | ans = self.base_domain.get_dmarc_has_aggregate_uri() 271 | return ans 272 | 273 | def get_dmarc_has_forensic_uri(self): 274 | """Check if this domain or its parent have a DMARC forensic URI.""" 275 | ans = self.dmarc_has_forensic_uri 276 | # If there are no forensic URIs then check the parents. 277 | if not ans and self.base_domain: 278 | ans = self.base_domain.get_dmarc_has_forensic_uri() 279 | return ans 280 | 281 | def get_dmarc_aggregate_uris(self): 282 | """Get this domain or its parent's DMARC aggregate URIs.""" 283 | ans = self.dmarc_aggregate_uris 284 | # If there are no aggregate URIs then check the parents. 285 | if not ans and self.base_domain: 286 | ans = self.base_domain.get_dmarc_aggregate_uris() 287 | return ans 288 | 289 | def get_dmarc_forensic_uris(self): 290 | """Get this domain or its parent's DMARC forensic URIs.""" 291 | ans = self.dmarc_forensic_uris 292 | # If there are no forensic URIs then check the parents. 293 | if not ans and self.base_domain: 294 | ans = self.base_domain.get_dmarc_forensic_uris() 295 | return ans 296 | 297 | def generate_results(self): 298 | """Generate the results for this domain.""" 299 | if len(self.starttls_results.keys()) == 0: 300 | domain_supports_smtp = None 301 | domain_supports_starttls = None 302 | mail_servers_that_support_smtp = None 303 | mail_servers_that_support_starttls = None 304 | else: 305 | mail_servers_that_support_smtp = [ 306 | x 307 | for x in self.starttls_results.keys() 308 | if self.starttls_results[x]["supports_smtp"] 309 | ] 310 | mail_servers_that_support_starttls = [ 311 | x 312 | for x in self.starttls_results.keys() 313 | if self.starttls_results[x]["starttls"] 314 | ] 315 | domain_supports_smtp = bool(mail_servers_that_support_smtp) 316 | domain_supports_starttls = domain_supports_smtp and all( 317 | [ 318 | self.starttls_results[x]["starttls"] 319 | for x in mail_servers_that_support_smtp 320 | ] 321 | ) 322 | 323 | results = OrderedDict( 324 | [ 325 | ("Domain", self.domain_name), 326 | ("Base Domain", self.base_domain_name), 327 | ("Live", self.is_live), 328 | ("MX Record", self.has_mail()), 329 | ("MX Record DNSSEC", self.mx_records_dnssec), 330 | ("Mail Servers", format_list(self.mail_servers)), 331 | ( 332 | "Mail Server Ports Tested", 333 | format_list([str(port) for port in self.ports_tested]), 334 | ), 335 | ( 336 | "Domain Supports SMTP Results", 337 | format_list(mail_servers_that_support_smtp), 338 | ), 339 | # True if and only if at least one mail server speaks SMTP 340 | ("Domain Supports SMTP", domain_supports_smtp), 341 | ( 342 | "Domain Supports STARTTLS Results", 343 | format_list(mail_servers_that_support_starttls), 344 | ), 345 | # True if and only if all mail servers that speak SMTP 346 | # also support STARTTLS 347 | ("Domain Supports STARTTLS", domain_supports_starttls), 348 | ("SPF Record", self.has_spf()), 349 | ("SPF Record DNSSEC", self.spf_dnssec), 350 | ("Valid SPF", self.valid_spf), 351 | ("SPF Results", format_list(self.spf)), 352 | ("DMARC Record", self.has_dmarc()), 353 | ("DMARC Record DNSSEC", self.dmarc_dnssec), 354 | ("Valid DMARC", self.has_dmarc() and self.valid_dmarc), 355 | ("DMARC Results", format_list(self.dmarc)), 356 | ("DMARC Record on Base Domain", self.parent_has_dmarc()), 357 | ("DMARC Record on Base Domain DNSSEC", self.parent_dmarc_dnssec()), 358 | ( 359 | "Valid DMARC Record on Base Domain", 360 | self.parent_has_dmarc() and self.parent_valid_dmarc(), 361 | ), 362 | ("DMARC Results on Base Domain", self.parent_dmarc_results()), 363 | ("DMARC Policy", self.get_dmarc_policy()), 364 | ("DMARC Subdomain Policy", self.get_dmarc_subdomain_policy()), 365 | ("DMARC Policy Percentage", self.get_dmarc_pct()), 366 | ( 367 | "DMARC Aggregate Report URIs", 368 | format_list(self.get_dmarc_aggregate_uris()), 369 | ), 370 | ( 371 | "DMARC Forensic Report URIs", 372 | format_list(self.get_dmarc_forensic_uris()), 373 | ), 374 | ("DMARC Has Aggregate Report URI", self.get_dmarc_has_aggregate_uri()), 375 | ("DMARC Has Forensic Report URI", self.get_dmarc_has_forensic_uri()), 376 | ( 377 | "DMARC Reporting Address Acceptance Error", 378 | self.dmarc_reports_address_error, 379 | ), 380 | ("Syntax Errors", format_list(self.syntax_errors)), 381 | ("Debug Info", format_list(self.debug_info)), 382 | ] 383 | ) 384 | 385 | return results 386 | -------------------------------------------------------------------------------- /src/trustymail/trustymail.py: -------------------------------------------------------------------------------- 1 | """Functions to check a domain's configuration for trustworthy mail.""" 2 | 3 | # Standard Python Libraries 4 | from collections import OrderedDict 5 | import csv 6 | import datetime 7 | import inspect 8 | import json 9 | import logging 10 | import re 11 | import smtplib 12 | import socket 13 | 14 | # Third-Party Libraries 15 | import DNS 16 | import dns.resolver 17 | import dns.reversename 18 | import requests 19 | import spf 20 | 21 | from .domain import Domain, get_public_suffix 22 | 23 | # A cache for SMTP scanning results 24 | _SMTP_CACHE = {} 25 | 26 | MAILTO_REGEX = re.compile( 27 | r"(mailto):([\w\-!#$%&'*+-/=?^_`{|}~][\w\-.!#$%&'*+-/=?^_`{|}~]*@[\w\-.]+)(!\w+)?" 28 | ) 29 | 30 | 31 | def domain_list_from_url(url): 32 | """Get a list of domains from a provided URL.""" 33 | if not url: 34 | return [] 35 | 36 | with requests.Session() as session: 37 | # Download current list of agencies, then let csv reader handle it. 38 | return domain_list_from_csv( 39 | session.get(url).content.decode("utf-8").splitlines() 40 | ) 41 | 42 | 43 | def domain_list_from_csv(csv_file): 44 | """Get a list of domains from a provided CSV file.""" 45 | domain_list = list(csv.reader(csv_file, delimiter=",")) 46 | 47 | # Check the headers for the word domain - use that column. 48 | 49 | domain_column = 0 50 | 51 | for i in range(0, len(domain_list[0])): 52 | header = domain_list[0][i] 53 | if "domain" in header.lower(): 54 | domain_column = i 55 | # CSV starts with headers, remove first row. 56 | domain_list.pop(0) 57 | break 58 | 59 | domains = [] 60 | for row in domain_list: 61 | if row is not None and len(row) > 0: 62 | domains.append(row[domain_column]) 63 | 64 | return domains 65 | 66 | 67 | def check_dnssec(domain, domain_name, record_type): 68 | """Test to see if a DNSSEC record is valid and correct. 69 | 70 | Checks a domain for DNSSEC whether the domain has a record of type that is protected 71 | by DNSSEC or NXDOMAIN or NoAnswer that is protected by DNSSEC. 72 | 73 | TODO: Probably does not follow redirects (CNAMEs). Should work on 74 | that in the future. 75 | """ 76 | try: 77 | query = dns.message.make_query(domain_name, record_type, want_dnssec=True) 78 | for nameserver in DNS_RESOLVERS: 79 | response = dns.query.tcp(query, nameserver, timeout=DNS_TIMEOUT) 80 | if response is not None: 81 | if response.flags & dns.flags.AD: 82 | return True 83 | else: 84 | return False 85 | except Exception as error: 86 | handle_error("[MX DNSSEC]", domain, error) 87 | return None 88 | 89 | 90 | def mx_scan(resolver, domain): 91 | """Scan a domain to see if it has any mail servers.""" 92 | try: 93 | if domain.mx_records is None: 94 | domain.mx_records = [] 95 | if domain.mail_servers is None: 96 | domain.mail_servers = [] 97 | # Use TCP, since we care about the content and correctness of the 98 | # records more than whether their records fit in a single UDP packet. 99 | for record in resolver.query(domain.domain_name, "MX", tcp=True): 100 | domain.add_mx_record(record) 101 | domain.mx_records_dnssec = check_dnssec(domain, domain.domain_name, "MX") 102 | except dns.resolver.NoNameservers as error: 103 | # The NoNameServers exception means that we got a SERVFAIL response. 104 | # These responses are almost always permanent, not temporary, so let's 105 | # treat the domain as not live. 106 | domain.is_live = False 107 | handle_error("[MX]", domain, error) 108 | except dns.resolver.NXDOMAIN as error: 109 | domain.is_live = False 110 | # NXDOMAIN can still have DNSSEC 111 | domain.mx_records_dnssec = check_dnssec(domain, domain.domain_name, "MX") 112 | handle_error("[MX]", domain, error) 113 | except dns.resolver.NoAnswer as error: 114 | # The NoAnswer exception means that the domain does exist in 115 | # DNS, but it does not have any MX records. It sort of makes 116 | # sense to treat this case as "not live", but @h-m-f-t 117 | # (Cameron Dixon) points out that "a domain not NXDOMAINing 118 | # or SERVFAILing is a reasonable proxy for existence. It's 119 | # functionally "live" if the domain resolves in public DNS, 120 | # and therefore can benefit from DMARC action." 121 | # 122 | # See also https://github.com/cisagov/trustymail/pull/91 123 | 124 | # NoAnswer can still have DNSSEC 125 | domain.mx_records_dnssec = check_dnssec(domain, domain.domain_name, "MX") 126 | handle_error("[MX]", domain, error) 127 | except dns.exception.Timeout as error: 128 | domain.mx_records_dnssec = check_dnssec(domain, domain.domain_name, "MX") 129 | handle_error("[MX]", domain, error) 130 | 131 | 132 | def starttls_scan(domain, smtp_timeout, smtp_localhost, smtp_ports, smtp_cache): 133 | """Scan a domain to see if it supports SMTP and supports STARTTLS. 134 | 135 | Scan a domain to see if it supports SMTP. If the domain does support 136 | SMTP, a further check will be done to see if it supports STARTTLS. 137 | All results are stored inside the Domain object that is passed in 138 | as a parameter. 139 | 140 | Parameters 141 | ---------- 142 | domain : Domain 143 | The Domain to be tested. 144 | 145 | smtp_timeout : int 146 | The SMTP connection timeout in seconds. 147 | 148 | smtp_localhost : str 149 | The hostname to use when connecting to SMTP servers. 150 | 151 | smtp_ports : obj:`list` of :obj:`str` 152 | A comma-delimited list of ports at which to look for SMTP servers. 153 | 154 | smtp_cache : bool 155 | Whether or not to cache SMTP results. 156 | """ 157 | for mail_server in domain.mail_servers: 158 | for port in smtp_ports: 159 | domain.ports_tested.add(port) 160 | server_and_port = mail_server + ":" + str(port) 161 | 162 | if not smtp_cache or (server_and_port not in _SMTP_CACHE): 163 | domain.starttls_results[server_and_port] = {} 164 | 165 | smtp_connection = smtplib.SMTP( 166 | timeout=smtp_timeout, local_hostname=smtp_localhost 167 | ) 168 | # The following line is useful when debugging why an 169 | # SMTP connection fails. It prints out all the 170 | # traffic sent to and from the SMTP server. 171 | smtp_connection.set_debuglevel(1) 172 | logging.debug("Testing " + server_and_port + " for STARTTLS support") 173 | 174 | # Look up the IPv4 address for mail_server. 175 | # 176 | # By default, smtplib looks for A and AAAA records 177 | # from DNS and uses the first one that it can connect 178 | # to. What I find when running in Lambda (at least in 179 | # my VPC that doesn't support IPv6) is that when DNS 180 | # returns IPv6 an address I get a low level "errno 97 181 | # - Address family not supported by protocol" error 182 | # and the other addresses returned by DNS are not 183 | # tried. Therefore the hostname is not scanned at 184 | # all. 185 | # 186 | # To get around this I look up the A record and use 187 | # that instead of the hostname in DNS when I call 188 | # smtp_connection.connect(). 189 | try: 190 | addr_info = socket.getaddrinfo( 191 | mail_server, port, socket.AF_INET, socket.SOCK_STREAM 192 | ) 193 | except socket.gaierror: 194 | # We get this exception if there is no A record 195 | # for the given mail server. This does happen, 196 | # since among their MX records some domains do 197 | # list some IPv6-only mail servers. 198 | # 199 | # Since we can't evaluate such cases we will 200 | # simply log this and give them credit. One of 201 | # the other mail servers will support IPv4. 202 | error_str = ( 203 | f"The mail server {mail_server} does not have an IPv4 address." 204 | ) 205 | handle_error("[STARTTLS]", domain, error_str) 206 | logging.warn(error_str) 207 | domain.starttls_results[server_and_port]["is_listening"] = True 208 | domain.starttls_results[server_and_port]["supports_smtp"] = True 209 | domain.starttls_results[server_and_port]["starttls"] = True 210 | continue 211 | 212 | # Extract the IP address from the socket addrinfo 213 | socket_address = addr_info[0][4] 214 | mail_server_ip_address = socket_address[0] 215 | 216 | # Try to connect. This will tell us if something is 217 | # listening. 218 | try: 219 | smtp_connection.connect(mail_server_ip_address, port) 220 | domain.starttls_results[server_and_port]["is_listening"] = True 221 | except ( 222 | socket.timeout, 223 | smtplib.SMTPConnectError, 224 | smtplib.SMTPServerDisconnected, 225 | ConnectionRefusedError, 226 | OSError, 227 | ) as error: 228 | handle_error("[STARTTLS]", domain, error) 229 | domain.starttls_results[server_and_port]["is_listening"] = False 230 | domain.starttls_results[server_and_port]["supports_smtp"] = False 231 | domain.starttls_results[server_and_port]["starttls"] = False 232 | 233 | if smtp_cache: 234 | _SMTP_CACHE[server_and_port] = domain.starttls_results[ 235 | server_and_port 236 | ] 237 | 238 | continue 239 | 240 | # Now try to say hello. This will tell us if the 241 | # thing that is listening is an SMTP server. 242 | try: 243 | smtp_connection.ehlo_or_helo_if_needed() 244 | domain.starttls_results[server_and_port]["supports_smtp"] = True 245 | logging.debug("\t Supports SMTP") 246 | except (smtplib.SMTPHeloError, smtplib.SMTPServerDisconnected) as error: 247 | handle_error("[STARTTLS]", domain, error) 248 | domain.starttls_results[server_and_port]["supports_smtp"] = False 249 | domain.starttls_results[server_and_port]["starttls"] = False 250 | # smtplib freaks out if you call quit on a non-open 251 | # connection 252 | try: 253 | smtp_connection.quit() 254 | except smtplib.SMTPServerDisconnected as error2: 255 | handle_error("[STARTTLS]", domain, error2) 256 | 257 | if smtp_cache: 258 | _SMTP_CACHE[server_and_port] = domain.starttls_results[ 259 | server_and_port 260 | ] 261 | 262 | continue 263 | 264 | # Now check if the server supports STARTTLS. 265 | has_starttls = smtp_connection.has_extn("STARTTLS") 266 | domain.starttls_results[server_and_port]["starttls"] = has_starttls 267 | logging.debug("\t Supports STARTTLS: " + str(has_starttls)) 268 | 269 | # Close the connection 270 | # smtplib freaks out if you call quit on a non-open 271 | # connection 272 | try: 273 | smtp_connection.quit() 274 | except smtplib.SMTPServerDisconnected as error: 275 | handle_error("[STARTTLS]", domain, error) 276 | 277 | # Copy the results into the cache, if necessary 278 | if smtp_cache: 279 | _SMTP_CACHE[server_and_port] = domain.starttls_results[ 280 | server_and_port 281 | ] 282 | else: 283 | logging.debug("\tUsing cached results for " + server_and_port) 284 | # Copy the cached results into the domain object 285 | domain.starttls_results[server_and_port] = _SMTP_CACHE[server_and_port] 286 | 287 | 288 | def check_spf_record(record_text, domain, strict=2): 289 | """Test to see if an SPF record is valid and correct. 290 | 291 | The record is tested by evaluating the response when we query 292 | using an IP that is known not to be a mail server that appears in 293 | the MX records for ANY domain. 294 | 295 | Parameters 296 | ---------- 297 | record_text : str 298 | The text of the SPF record to be tested. 299 | 300 | domain : trustymail.Domain 301 | The Domain object corresponding to the SPF record being 302 | tested. Any errors will be logged to this object. 303 | 304 | strict : bool or int 305 | The level of strictness to use when verifying an SPF record. 306 | Valid values are True, False, and 2. The last value is the 307 | most harsh. 308 | 309 | """ 310 | try: 311 | # Here I am using the IP address for 312 | # ec2-100-27-42-254.compute-1.amazonaws.com (100.27.42.254) 313 | # since it (1) has a valid PTR record and (2) is not listed by 314 | # anyone as a valid mail server. (The second item follows 315 | # from the fact that AWS has semi-permanently assigned this IP 316 | # to NCATS as part of our contiguous netblock, and we are not 317 | # using it as a mail server or including it as an MX record 318 | # for any domain.) 319 | # 320 | # Passing verbose=True causes the SPF library being used to 321 | # print out the SPF records encountered as include and 322 | # redirect cause other SPF records to be looked up. 323 | query = spf.query( 324 | "100.27.42.254", 325 | "email_wizard@" + domain.domain_name, 326 | domain.domain_name, 327 | strict=strict, 328 | verbose=True, 329 | ) 330 | response = query.check(spf=record_text) 331 | 332 | response_type = response[0] 333 | # A value of none means that no valid SPF record was obtained 334 | # from DNS. We get this result when we get an ambiguous 335 | # result because of an SPF record with incorrect syntax, then 336 | # rerun check_spf_record() with strict=True (instead of 2). 337 | if ( 338 | response_type == "temperror" 339 | or response_type == "permerror" 340 | or response_type == "none" 341 | ): 342 | domain.valid_spf = False 343 | handle_error( 344 | "[SPF]", 345 | domain, 346 | "SPF query returned {}: {}".format(response_type, response[2]), 347 | ) 348 | elif response_type == "ambiguous": 349 | # Log the ambiguity so it appears in the results CSV 350 | handle_error( 351 | "[SPF]", 352 | domain, 353 | "SPF query returned {}: {}".format(response_type, response[2]), 354 | ) 355 | 356 | # Now rerun the check with less strictness to get an 357 | # actual result. (With strict=2, the SPF library stops 358 | # processing once it encounters an AmbiguityWarning.) 359 | check_spf_record(record_text, domain, True) 360 | else: 361 | # Everything checks out. The SPF syntax seems valid. 362 | domain.valid_spf = True 363 | except spf.AmbiguityWarning as error: 364 | domain.valid_spf = False 365 | handle_error("[SPF]", domain, error) 366 | 367 | 368 | def get_spf_record_text(resolver, domain_name, domain, follow_redirect=False): 369 | """Get the SPF record text for the given domain name. 370 | 371 | DNS queries are performed using the dns.resolver.Resolver object. 372 | Errors are logged to the trustymail.Domain object. The Boolean 373 | parameter indicates whether to follow redirects in SPF records. 374 | 375 | Parameters 376 | ---------- 377 | resolver : dns.resolver.Resolver 378 | The Resolver object to use for DNS queries. 379 | 380 | domain_name : str 381 | The domain name to query for an SPF record. 382 | 383 | domain : trustymail.Domain 384 | The Domain object whose corresponding SPF record text is 385 | desired. Any errors will be logged to this object. 386 | 387 | follow_redirect : bool 388 | A Boolean value indicating whether to follow redirects in SPF 389 | records. 390 | 391 | Returns 392 | ------- 393 | str: The desired SPF record text 394 | """ 395 | record_to_return = None 396 | try: 397 | # Use TCP, since we care about the content and correctness of the 398 | # records more than whether their records fit in a single UDP packet. 399 | for record in resolver.query(domain_name, "TXT", tcp=True): 400 | record_text = remove_quotes(record.to_text()) 401 | 402 | if not record_text.startswith("v=spf1"): 403 | # Not an spf record, ignore it. 404 | continue 405 | 406 | match = re.search(r"v=spf1\s*redirect=(\S*)", record_text) 407 | if follow_redirect and match: 408 | redirect_domain_name = match.group(1) 409 | record_to_return = get_spf_record_text( 410 | resolver, redirect_domain_name, domain 411 | ) 412 | else: 413 | record_to_return = record_text 414 | 415 | domain.spf_dnssec = check_dnssec(domain, domain.domain_name, "TXT") 416 | except dns.resolver.NoNameservers as error: 417 | # The NoNameservers exception means that we got a SERVFAIL response. 418 | # These responses are almost always permanent, not temporary, so let's 419 | # treat the domain as not live. 420 | domain.is_live = False 421 | handle_error("[SPF]", domain, error) 422 | except dns.resolver.NXDOMAIN as error: 423 | domain.is_live = False 424 | domain.spf_dnssec = check_dnssec(domain, domain.domain_name, "TXT") 425 | handle_error("[SPF]", domain, error) 426 | except dns.resolver.NoAnswer as error: 427 | domain.spf_dnssec = check_dnssec(domain, domain.domain_name, "TXT") 428 | handle_error("[SPF]", domain, error) 429 | except dns.exception.Timeout as error: 430 | domain.spf_dnssec = check_dnssec(domain, domain.domain_name, "TXT") 431 | handle_error("[SPF]", domain, error) 432 | return record_to_return 433 | 434 | 435 | def spf_scan(resolver, domain): 436 | """Scan a domain to see if it supports SPF. 437 | 438 | If the domain has an SPF record, verify that it properly handles mail sent from 439 | an IP known not to be listed in an MX record for ANY domain. 440 | 441 | Parameters 442 | ---------- 443 | resolver : dns.resolver.Resolver 444 | The Resolver object to use for DNS queries. 445 | 446 | domain : trustymail.Domain 447 | The Domain object being scanned for SPF support. Any errors 448 | will be logged to this object. 449 | 450 | """ 451 | if domain.spf is None: 452 | domain.spf = [] 453 | 454 | # If an SPF record exists, record the raw SPF record text in the 455 | # Domain object 456 | record_text_not_following_redirect = get_spf_record_text( 457 | resolver, domain.domain_name, domain 458 | ) 459 | if record_text_not_following_redirect: 460 | domain.spf.append(record_text_not_following_redirect) 461 | 462 | record_text_following_redirect = get_spf_record_text( 463 | resolver, domain.domain_name, domain, True 464 | ) 465 | if record_text_following_redirect: 466 | check_spf_record(record_text_following_redirect, domain) 467 | 468 | 469 | def parse_dmarc_report_uri(uri): 470 | """ 471 | Parse a DMARC Reporting (i.e. ``rua``/``ruf``) URI. 472 | 473 | Notes 474 | ----- 475 | ``mailto:`` is the only reporting URI supported in `DMARC1` 476 | 477 | Arguments 478 | --------- 479 | uri: A DMARC URI 480 | 481 | Returns 482 | ------- 483 | OrderedDict: Keys: ''scheme`` ``address`` and ``size_limit`` 484 | 485 | """ 486 | uri = uri.strip() 487 | mailto_matches = MAILTO_REGEX.findall(uri) 488 | if len(mailto_matches) != 1: 489 | return None 490 | match = mailto_matches[0] 491 | scheme = match[0] 492 | email_address = match[1] 493 | size_limit = match[2].lstrip("!") 494 | if size_limit == "": 495 | size_limit = None 496 | 497 | return OrderedDict( 498 | [("scheme", scheme), ("address", email_address), ("size_limit", size_limit)] 499 | ) 500 | 501 | 502 | def dmarc_scan(resolver, domain): 503 | """Scan a domain to see if it supports DMARC.""" 504 | # dmarc records are kept in TXT records for _dmarc.domain_name. 505 | try: 506 | if domain.dmarc is None: 507 | domain.dmarc = [] 508 | dmarc_domain = "_dmarc.%s" % domain.domain_name 509 | # Use TCP, since we care about the content and correctness of the 510 | # records more than whether their records fit in a single UDP packet. 511 | all_records = resolver.query(dmarc_domain, "TXT", tcp=True) 512 | domain.dmarc_dnssec = check_dnssec(domain, dmarc_domain, "TXT") 513 | # According to step 4 in section 6.6.3 of the RFC 514 | # (https://tools.ietf.org/html/rfc7489#section-6.6.3), "Records that do 515 | # not start with a "v=" tag that identifies the current version of 516 | # DMARC are discarded." 517 | records = [ 518 | record 519 | for record in all_records 520 | if record.to_text().startswith('"v=DMARC1;') 521 | ] 522 | 523 | # Treat multiple DMARC records as an error, in accordance with the RFC 524 | # (https://tools.ietf.org/html/rfc7489#section-6.6.3) 525 | if len(records) > 1: 526 | handle_error("[DMARC]", domain, "Warning: Multiple DMARC records present") 527 | domain.valid_dmarc = False 528 | elif records: 529 | record = records[0] 530 | 531 | record_text = remove_quotes(record.to_text()) 532 | 533 | # Ensure the record is a DMARC record. Some domains that 534 | # redirect will cause an SPF record to show. 535 | if record_text.startswith("v=DMARC1"): 536 | domain.dmarc.append(record_text) 537 | elif record_text.startswith("v=spf1"): 538 | msg = ( 539 | "Found a SPF record where a DMARC record should be; most likely, the _dmarc " 540 | "subdomain record does not actually exist, and the request for TXT records was " 541 | "redirected to the base domain" 542 | ) 543 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 544 | domain.valid_dmarc = False 545 | 546 | # Remove excess whitespace 547 | record_text = record_text.strip() 548 | 549 | # DMARC records follow a specific outline as to how they are 550 | # defined - tag:value We can split this up into a easily 551 | # manipulatable dictionary 552 | tag_dict = {} 553 | for options in record_text.split(";"): 554 | if "=" not in options: 555 | continue 556 | tag = options.split("=")[0].strip() 557 | value = options.split("=")[1].strip() 558 | tag_dict[tag] = value 559 | 560 | # Before we set sp=p if it is not explicitly contained in 561 | # the DMARC record, log a warning if it is explicitly set 562 | # for a subdomain of an organizational domain. 563 | if "sp" in tag_dict and not domain.is_base_domain: 564 | handle_error( 565 | "[DMARC]", 566 | domain, 567 | "Warning: The sp tag will be ignored for DMARC records published on subdomains. See here for details: https://tools.ietf.org/html/rfc7489#section-6.3.", 568 | syntax_error=False, 569 | ) 570 | if "p" not in tag_dict: 571 | msg = "Record missing required policy (p) tag" 572 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 573 | domain.valid_dmarc = False 574 | elif "sp" not in tag_dict: 575 | tag_dict["sp"] = tag_dict["p"] 576 | if "ri" not in tag_dict: 577 | tag_dict["ri"] = 86400 578 | if "pct" not in tag_dict: 579 | tag_dict["pct"] = 100 580 | if "adkim" not in tag_dict: 581 | tag_dict["adkim"] = "r" 582 | if "aspf" not in tag_dict: 583 | tag_dict["aspf"] = "r" 584 | if "fo" not in tag_dict: 585 | tag_dict["fo"] = "0" 586 | if "rf" not in tag_dict: 587 | tag_dict["rf"] = "afrf" 588 | if "rua" not in tag_dict: 589 | domain.dmarc_has_aggregate_uri = False 590 | if "ruf" not in tag_dict: 591 | domain.dmarc_has_forensic_uri = False 592 | 593 | for tag in tag_dict: 594 | if tag not in [ 595 | "v", 596 | "mailto", 597 | "rf", 598 | "p", 599 | "sp", 600 | "adkim", 601 | "aspf", 602 | "fo", 603 | "pct", 604 | "ri", 605 | "rua", 606 | "ruf", 607 | ]: 608 | msg = "Unknown DMARC tag {}".format(tag) 609 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 610 | domain.valid_dmarc = False 611 | elif tag == "p": 612 | if tag_dict[tag] not in ["none", "quarantine", "reject"]: 613 | msg = "Unknown DMARC policy {}".format(tag) 614 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 615 | domain.valid_dmarc = False 616 | else: 617 | domain.dmarc_policy = tag_dict[tag] 618 | elif tag == "sp": 619 | if tag_dict[tag] not in ["none", "quarantine", "reject"]: 620 | msg = "Unknown DMARC subdomain policy {}".format(tag_dict[tag]) 621 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 622 | domain.valid_dmarc = False 623 | else: 624 | domain.dmarc_subdomain_policy = tag_dict[tag] 625 | elif tag == "fo": 626 | values = tag_dict[tag].split(":") 627 | if "0" in values and "1" in values: 628 | msg = "fo tag values 0 and 1 are mutually exclusive" 629 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 630 | for value in values: 631 | if value not in ["0", "1", "d", "s"]: 632 | msg = "Unknown DMARC fo tag value {}".format(value) 633 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 634 | domain.valid_dmarc = False 635 | elif tag == "rf": 636 | values = tag_dict[tag].split(":") 637 | for value in values: 638 | if value not in ["afrf"]: 639 | msg = "Unknown DMARC rf tag value {}".format(value) 640 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 641 | domain.valid_dmarc = False 642 | elif tag == "ri": 643 | try: 644 | int(tag_dict[tag]) 645 | except ValueError: 646 | msg = "Invalid DMARC ri tag value: {} - must be an integer".format( 647 | tag_dict[tag] 648 | ) 649 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 650 | domain.valid_dmarc = False 651 | elif tag == "pct": 652 | try: 653 | pct = int(tag_dict[tag]) 654 | if pct < 0 or pct > 100: 655 | msg = ( 656 | "Error: invalid DMARC pct tag value: {} - must be an integer between " 657 | "0 and 100".format(tag_dict[tag]) 658 | ) 659 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 660 | domain.valid_dmarc = False 661 | domain.dmarc_pct = pct 662 | if pct < 100: 663 | handle_syntax_error( 664 | "[DMARC]", 665 | domain, 666 | "Warning: The DMARC pct tag value may be less than 100 (the implicit default) during deployment, but should be removed or set to 100 upon full deployment", 667 | ) 668 | except ValueError: 669 | msg = "invalid DMARC pct tag value: {} - must be an integer".format( 670 | tag_dict[tag] 671 | ) 672 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 673 | domain.valid_dmarc = False 674 | elif tag == "rua" or tag == "ruf": 675 | uris = tag_dict[tag].split(",") 676 | if len(uris) > 2: 677 | handle_error( 678 | "[DMARC]", 679 | domain, 680 | "Warning: The {} tag specifies {} URIs. Receivers are not required to send reports to more than two URIs - https://tools.ietf.org/html/rfc7489#section-6.2.".format( 681 | tag, len(uris) 682 | ), 683 | syntax_error=False, 684 | ) 685 | for uri in uris: 686 | # mailto: is currently the only type of DMARC URI 687 | parsed_uri = parse_dmarc_report_uri(uri) 688 | if parsed_uri is None: 689 | msg = "Error: {} is an invalid DMARC URI".format(uri) 690 | handle_syntax_error("[DMARC]", domain, "{}".format(msg)) 691 | domain.valid_dmarc = False 692 | else: 693 | if tag == "rua": 694 | domain.dmarc_aggregate_uris.append(uri) 695 | elif tag == "ruf": 696 | domain.dmarc_forensic_uris.append(uri) 697 | email_address = parsed_uri["address"] 698 | email_domain = email_address.split("@")[-1] 699 | if ( 700 | get_public_suffix(email_domain).lower() 701 | != domain.base_domain_name.lower() 702 | ): 703 | target = "{}._report._dmarc.{}".format( 704 | domain.domain_name, email_domain 705 | ) 706 | error_message = ( 707 | "{} does not indicate that it accepts DMARC reports about {} - " 708 | "https://tools.ietf.org" 709 | "/html/rfc7489#section-7.1".format( 710 | email_domain, domain.domain_name 711 | ) 712 | ) 713 | try: 714 | answer = remove_quotes( 715 | resolver.query(target, "TXT", tcp=True)[ 716 | 0 717 | ].to_text() 718 | ) 719 | if not answer.startswith("v=DMARC1"): 720 | handle_error( 721 | "[DMARC]", 722 | domain, 723 | "{}".format(error_message), 724 | ) 725 | domain.dmarc_reports_address_error = True 726 | domain.valid_dmarc = False 727 | except ( 728 | dns.resolver.NXDOMAIN, 729 | dns.resolver.NoAnswer, 730 | dns.resolver.NoNameservers, 731 | dns.exception.Timeout, 732 | ): 733 | handle_syntax_error( 734 | "[DMARC]", domain, "{}".format(error_message) 735 | ) 736 | domain.dmarc_reports_address_error = True 737 | domain.valid_dmarc = False 738 | try: 739 | # Ensure ruf/rua/email domains have MX records 740 | resolver.query(email_domain, "MX", tcp=True) 741 | except ( 742 | dns.resolver.NXDOMAIN, 743 | dns.resolver.NoAnswer, 744 | dns.resolver.NoNameservers, 745 | dns.exception.Timeout, 746 | ): 747 | handle_syntax_error( 748 | "[DMARC]", 749 | domain, 750 | "The domain for reporting " 751 | "address {} does not have any " 752 | "MX records".format(email_address), 753 | ) 754 | domain.valid_dmarc = False 755 | 756 | # Log a warning if the DMARC record specifies a policy but does not 757 | # specify any ruf or rua URIs, since this greatly reduces the 758 | # usefulness of DMARC. 759 | if "p" in tag_dict and "rua" not in tag_dict and "ruf" not in tag_dict: 760 | handle_syntax_error( 761 | "[DMARC]", 762 | domain, 763 | "Warning: A DMARC policy is specified but no reporting URIs. This makes the DMARC implementation considerably less useful than it could be. See https://tools.ietf.org/html/rfc7489#section-6.5 for more details.", 764 | ) 765 | 766 | domain.dmarc_has_aggregate_uri = len(domain.dmarc_aggregate_uris) > 0 767 | domain.dmarc_has_forensic_uri = len(domain.dmarc_forensic_uris) > 0 768 | except ( 769 | dns.resolver.NoAnswer, 770 | dns.resolver.NXDOMAIN, 771 | dns.exception.Timeout, 772 | ) as error: 773 | domain.dmarc_dnssec = check_dnssec(domain, dmarc_domain, "TXT") 774 | handle_error("[DMARC]", domain, error) 775 | except dns.resolver.NoNameservers as error: 776 | # Normally we count a NoNameservers exception as indicating 777 | # that a domain is "not live". In this case we don't, though, 778 | # since the DMARC DNS check doesn't query for the domain name 779 | # itself. If the domain name is domain.com, the DMARC DNS 780 | # check queries for _dmarc.domain.com. 781 | handle_error("[DMARC]", domain, error) 782 | 783 | 784 | def find_host_from_ip(resolver, ip_addr): 785 | """Find the host name for a given IP address.""" 786 | # Use TCP, since we care about the content and correctness of the records 787 | # more than whether their records fit in a single UDP packet. 788 | hostname, _ = resolver.query(dns.reversename.from_address(ip_addr), "PTR", tcp=True) 789 | return str(hostname) 790 | 791 | 792 | def scan( 793 | domain_name, 794 | timeout, 795 | smtp_timeout, 796 | smtp_localhost, 797 | smtp_ports, 798 | smtp_cache, 799 | scan_types, 800 | dns_hostnames, 801 | ): 802 | """Parse a domain's DNS information for mail related records.""" 803 | # 804 | # Configure the dnspython library 805 | # 806 | global DNS_TIMEOUT, DNS_RESOLVERS 807 | 808 | # Our resolver 809 | # 810 | # Note that it uses the system configuration in /etc/resolv.conf 811 | # if no DNS hostnames are specified. 812 | resolver = dns.resolver.Resolver(configure=not dns_hostnames) 813 | # This is a setting that controls whether we retry DNS servers if 814 | # we receive a SERVFAIL response from them. We set this to False 815 | # because, unless the reason for the SERVFAIL is truly temporary 816 | # and resolves before trustymail finishes scanning the domain, 817 | # this can obscure the potentially informative SERVFAIL error as a 818 | # DNS timeout because of the way dns.resolver.query() is written. 819 | # See 820 | # http://www.dnspython.org/docs/1.14.0/dns.resolver-pysrc.html#Resolver.query. 821 | resolver.retry_servfail = False 822 | # Set some timeouts. The timeout should be less than or equal to 823 | # the lifetime, but longer than the time a DNS server takes to 824 | # return a SERVFAIL (since otherwise it's possible to get a DNS 825 | # timeout when you should be getting a SERVFAIL.) See 826 | # http://www.dnspython.org/docs/1.14.0/dns.resolver-pysrc.html#Resolver.query 827 | # and 828 | # http://www.dnspython.org/docs/1.14.0/dns.resolver-pysrc.html#Resolver._compute_timeout. 829 | resolver.timeout = float(timeout) 830 | resolver.lifetime = float(timeout) 831 | DNS_TIMEOUT = timeout 832 | # If the user passed in DNS hostnames to query against then use them 833 | if dns_hostnames: 834 | resolver.nameservers = dns_hostnames 835 | DNS_RESOLVERS = dns_hostnames 836 | else: 837 | DNS_RESOLVERS = resolver.nameservers 838 | 839 | # 840 | # The spf library uses py3dns behind the scenes, so we need to configure 841 | # that too 842 | # 843 | DNS.defaults["timeout"] = timeout 844 | # Use TCP instead of UDP 845 | DNS.defaults["protocol"] = "tcp" 846 | # If the user passed in DNS hostnames to query against then use them 847 | if dns_hostnames: 848 | DNS.defaults["server"] = dns_hostnames 849 | 850 | # Domain's constructor needs all these parameters because it does a DMARC 851 | # scan in its init 852 | domain = Domain( 853 | domain_name, 854 | timeout, 855 | smtp_timeout, 856 | smtp_localhost, 857 | smtp_ports, 858 | smtp_cache, 859 | dns_hostnames, 860 | ) 861 | 862 | logging.debug("[{}]".format(domain_name.lower())) 863 | 864 | if scan_types["mx"] and domain.is_live: 865 | mx_scan(resolver, domain) 866 | 867 | if scan_types["starttls"] and domain.is_live: 868 | starttls_scan(domain, smtp_timeout, smtp_localhost, smtp_ports, smtp_cache) 869 | 870 | if scan_types["spf"] and domain.is_live: 871 | spf_scan(resolver, domain) 872 | 873 | if scan_types["dmarc"] and domain.is_live: 874 | dmarc_scan(resolver, domain) 875 | 876 | # If the user didn't specify any scans then run a full scan. 877 | if domain.is_live and not ( 878 | scan_types["mx"] 879 | or scan_types["starttls"] 880 | or scan_types["spf"] 881 | or scan_types["dmarc"] 882 | ): 883 | mx_scan(resolver, domain) 884 | starttls_scan(domain, smtp_timeout, smtp_localhost, smtp_ports, smtp_cache) 885 | spf_scan(resolver, domain) 886 | dmarc_scan(resolver, domain) 887 | 888 | return domain 889 | 890 | 891 | def handle_error(prefix, domain, error, syntax_error=False): 892 | """Handle the provided error by logging a message and storing it in the Domain object. 893 | 894 | Logging is performed via the Python logging library and recording it in the 895 | debug_info or syntax_error members of the trustymail.Domain object. 896 | 897 | Since the "Debug Info" and "Syntax Error" fields in the CSV output 898 | of trustymail come directly from the debug_info and syntax_error 899 | members of the trustymail.Domain object, and that CSV is likely 900 | all we will have to reconstruct how trustymail reached the 901 | conclusions it did, it is vital to record as much helpful 902 | information as possible. 903 | 904 | Parameters 905 | ---------- 906 | prefix : str 907 | The prefix to use when constructing the log string. This is 908 | usually the type of trustymail test that was being performed 909 | when the error condition occurred. 910 | 911 | domain : trustymail.Domain 912 | The Domain object in which the error or syntax error should be 913 | recorded. 914 | 915 | error : str, BaseException, or Exception 916 | Either a string describing the error, or an exception object 917 | representing the error. 918 | 919 | syntax_error : bool 920 | If True then the error will be recorded in the syntax_error 921 | member of the trustymail.Domain object. Otherwise it is 922 | recorded in the error member of the trustymail.Domain object. 923 | """ 924 | # Get the previous frame in the stack - the one that is calling 925 | # this function 926 | frame = inspect.currentframe().f_back 927 | function = frame.f_code 928 | function_name = function.co_name 929 | filename = function.co_filename 930 | line = frame.f_lineno 931 | 932 | error_template = "{prefix} In {function_name} at {filename}:{line}: {error}" 933 | 934 | if hasattr(error, "message"): 935 | if syntax_error and "NXDOMAIN" in error.message and prefix != "[DMARC]": 936 | domain.is_live = False 937 | error_string = error_template.format( 938 | prefix=prefix, 939 | function_name=function_name, 940 | line=line, 941 | filename=filename, 942 | error=error.message, 943 | ) 944 | else: 945 | error_string = error_template.format( 946 | prefix=prefix, 947 | function_name=function_name, 948 | line=line, 949 | filename=filename, 950 | error=str(error), 951 | ) 952 | 953 | if syntax_error: 954 | domain.syntax_errors.append(error_string) 955 | else: 956 | domain.debug_info.append(error_string) 957 | logging.debug(error_string) 958 | 959 | 960 | def handle_syntax_error(prefix, domain, error): 961 | """Handle a syntax error by passing it to handle_error().""" 962 | handle_error(prefix, domain, error, syntax_error=True) 963 | 964 | 965 | def generate_csv(domains, file_name): 966 | """Generate a CSV file with the given domain information.""" 967 | with open(file_name, "w", encoding="utf-8", newline="\n") as output_file: 968 | writer = csv.DictWriter( 969 | output_file, fieldnames=domains[0].generate_results().keys() 970 | ) 971 | 972 | # First row should always be the headers 973 | writer.writeheader() 974 | 975 | for domain in domains: 976 | writer.writerow(domain.generate_results()) 977 | output_file.flush() 978 | 979 | 980 | def generate_json(domains): 981 | """Generate a JSON string with the given domain information.""" 982 | output = [] 983 | for domain in domains: 984 | output.append(domain.generate_results()) 985 | 986 | return json.dumps(output, indent=2, default=format_datetime) 987 | 988 | 989 | # Taken from pshtt to keep formatting similar 990 | def format_datetime(obj): 991 | """Format the provided datetime information.""" 992 | if isinstance(obj, datetime.date): 993 | return obj.isoformat() 994 | elif isinstance(obj, str): 995 | return obj 996 | else: 997 | return None 998 | 999 | 1000 | def remove_quotes(txt_record): 1001 | """Remove double quotes and contatenate strings in a DNS TXT record. 1002 | 1003 | A DNS TXT record can contain multiple double-quoted strings, and 1004 | in that case the client has to remove the quotes and concatenate the 1005 | strings. This function does just that. 1006 | 1007 | Parameters 1008 | ---------- 1009 | txt_record : str 1010 | The DNS TXT record that possibly consists of multiple 1011 | double-quoted strings. 1012 | 1013 | Returns 1014 | ------- 1015 | str: The DNS TXT record with double-quoted strings unquoted and 1016 | concatenated. 1017 | """ 1018 | # This regular expression removes leading and trailing double quotes and 1019 | # also removes any pairs of double quotes separated by one or more spaces. 1020 | return re.sub('^"|"$|" +"', "", txt_record) 1021 | -------------------------------------------------------------------------------- /tag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o nounset 4 | set -o errexit 5 | set -o pipefail 6 | 7 | version=$(./bump_version.sh show) 8 | 9 | git tag "v$version" && git push --tags 10 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """pytest plugin configuration. 2 | 3 | https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins 4 | """ 5 | 6 | # Third-Party Libraries 7 | import pytest 8 | 9 | 10 | def pytest_addoption(parser): 11 | """Add new commandline options to pytest.""" 12 | parser.addoption( 13 | "--runslow", action="store_true", default=False, help="run slow tests" 14 | ) 15 | 16 | 17 | def pytest_configure(config): 18 | """Register new markers.""" 19 | config.addinivalue_line("markers", "slow: mark test as slow") 20 | 21 | 22 | def pytest_collection_modifyitems(config, items): 23 | """Modify collected tests based on custom marks and commandline options.""" 24 | if config.getoption("--runslow"): 25 | # --runslow given in cli: do not skip slow tests 26 | return 27 | skip_slow = pytest.mark.skip(reason="need --runslow option to run") 28 | for item in items: 29 | if "slow" in item.keywords: 30 | item.add_marker(skip_slow) 31 | -------------------------------------------------------------------------------- /tests/test_trustymail.py: -------------------------------------------------------------------------------- 1 | """Tests for the trustymail module.""" 2 | 3 | # Standard Python Libraries 4 | import unittest 5 | 6 | 7 | class TestLiveliness(unittest.TestCase): 8 | """Test the liveliness of a domain.""" 9 | 10 | def test_domain_list_parsing(self): 11 | """Test that a domain list is correctly parsed.""" 12 | pass 13 | --------------------------------------------------------------------------------