├── .flake8 ├── .github ├── dependabot.yml └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── ci-setup-golang.sh ├── gotfparse ├── README.md ├── cmd │ ├── tfdump │ │ └── main.go │ ├── tfparse │ │ └── main.go │ └── tftest │ │ ├── README.md │ │ └── main.go ├── go.mod ├── go.sum └── pkg │ └── converter │ ├── converter.go │ ├── hacks.go │ ├── interface.go │ ├── options.go │ └── relativeResolveFs.go ├── justfile ├── pyproject.toml ├── requirements-dev.txt ├── setup.py ├── tests ├── terraform │ ├── apply-time-vals │ │ └── main.tf │ ├── apprunner │ │ └── main.tf │ ├── dynamic-stuff │ │ └── main.tf │ ├── ec2-tags │ │ └── main.tf │ ├── eks │ │ ├── eks-nodegroup.tf │ │ ├── main.tf │ │ ├── network.tf │ │ └── providers.tf │ ├── func-check │ │ ├── lambdas │ │ │ ├── abc │ │ │ │ └── main.go │ │ │ └── xyz │ │ │ │ └── main.go │ │ └── root │ │ │ ├── files │ │ │ ├── x.py │ │ │ └── y.py │ │ │ ├── main.tf │ │ │ ├── modules │ │ │ ├── x │ │ │ │ └── main.tf │ │ │ ├── y │ │ │ │ └── main.tf │ │ │ └── z │ │ │ │ └── main.tf │ │ │ └── readme.md │ ├── local-module-above-root │ │ ├── module │ │ │ └── main.tf │ │ └── root │ │ │ └── main.tf │ ├── module-in-out-nested │ │ ├── main.tf │ │ └── module │ │ │ ├── bucket │ │ │ └── main.tf │ │ │ └── tags │ │ │ └── base │ │ │ └── main.tf │ ├── module-in-out │ │ ├── main.tf │ │ └── module │ │ │ ├── bucket │ │ │ └── main.tf │ │ │ └── tags │ │ │ └── main.tf │ ├── moved │ │ └── main.tf │ ├── notify_slack │ │ └── main.tf │ ├── references │ │ └── main.tf │ ├── variables-and-locals │ │ └── main.tf │ ├── vars-bad-types │ │ ├── main.tf │ │ ├── numbers.tfvars │ │ └── strings.tfvars │ ├── vars-file │ │ ├── example.tfvars │ │ ├── file.tf │ │ ├── terraform.tfstate │ │ └── vars.tf │ ├── vpc_module │ │ └── main.tf │ └── workspace │ │ └── main.tf └── test_tfparse.py └── tfparse ├── __init__.py └── build_cffi.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 100 3 | exclude = 4 | _tfparse.py -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "monthly" 8 | commit-message: 9 | prefix: ":seedling:" 10 | groups: 11 | github-actions: 12 | patterns: 13 | - "*" 14 | 15 | - package-ecosystem: "pip" 16 | directory: "/" 17 | schedule: 18 | interval: "monthly" 19 | commit-message: 20 | prefix: ":seedling:" 21 | groups: 22 | python-requirements: 23 | patterns: 24 | - "*" 25 | 26 | - package-ecosystem: "gomod" 27 | directory: "/" 28 | schedule: 29 | interval: "monthly" 30 | commit-message: 31 | prefix: ":seedling:" 32 | groups: 33 | golang-requirements: 34 | patterns: 35 | - "*" 36 | 37 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | workflow_dispatch: 7 | inputs: {} 8 | env: 9 | CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-*" 10 | jobs: 11 | 12 | Build-Linux: 13 | strategy: 14 | matrix: 15 | include: 16 | - runner: ubuntu-latest 17 | cibw_arch: aarch64 18 | - runner: ubuntu-latest 19 | cibw_arch: x86_64 20 | runs-on: ${{ matrix.runner }} 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 24 | - name: Set up QEMU 25 | uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 26 | with: 27 | platforms: arm64 28 | - name: Build wheels 29 | uses: pypa/cibuildwheel@faf86a6ed7efa889faf6996aa23820831055001a 30 | env: 31 | CIBW_ENVIRONMENT: PATH=$(pwd)/go/bin:$PATH 32 | CIBW_BEFORE_ALL: sh ci-setup-golang.sh 33 | CIBW_SKIP: "*musllinux*" 34 | CIBW_ARCHS: ${{ matrix.cibw_arch }} 35 | - name: Upload Artifacts 36 | uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 37 | with: 38 | name: wheels-linux-${{ matrix.cibw_arch }} 39 | path: ./wheelhouse/*.whl 40 | 41 | Build-Windows: 42 | runs-on: windows-latest 43 | steps: 44 | - name: Checkout 45 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 46 | - name: Set up Go 47 | uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b 48 | with: 49 | go-version: "1.21.5" 50 | cache: true 51 | cache-dependency-path: "gotfparse/go.sum" 52 | - name: Build wheels 53 | uses: pypa/cibuildwheel@faf86a6ed7efa889faf6996aa23820831055001a 54 | env: 55 | CGO_ENABLED: 1 56 | CIBW_ARCHS: AMD64 57 | - name: Upload Artifacts 58 | uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 59 | with: 60 | name: wheels-windows 61 | path: ./wheelhouse/*.whl 62 | 63 | Build-MacOS: 64 | strategy: 65 | matrix: 66 | include: 67 | - cibw_arch: "x86_64" 68 | go_arch: "amd64" 69 | - cibw_arch: "arm64" 70 | go_arch: "arm64" 71 | runs-on: macos-latest 72 | steps: 73 | - name: Checkout 74 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 75 | - name: Set up Go 76 | uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b 77 | with: 78 | go-version: "1.21.5" 79 | cache: true 80 | cache-dependency-path: "gotfparse/go.sum" 81 | - name: Build wheels 82 | uses: pypa/cibuildwheel@faf86a6ed7efa889faf6996aa23820831055001a 83 | env: 84 | CGO_ENABLED: 1 85 | CIBW_ARCHS: ${{ matrix.cibw_arch }} 86 | GOARCH: ${{ matrix.go_arch }} 87 | - name: Upload Artifacts 88 | uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 89 | with: 90 | name: wheels-macos-${{ matrix.cibw_arch }} 91 | path: ./wheelhouse/*.whl 92 | 93 | Gather: 94 | needs: [Build-Linux, Build-MacOS, Build-Windows] 95 | runs-on: ubuntu-latest 96 | outputs: 97 | hash: ${{ steps.hash.outputs.hash }} 98 | steps: 99 | - name: Fetch Wheels 100 | uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 101 | with: 102 | pattern: "wheels-*" 103 | path: dist 104 | merge-multiple: true 105 | - name: Display downloaded artifacts 106 | run: ls -lh dist 107 | - name: Generate Hashes 108 | id: hash 109 | run: cd dist && echo "hash=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT 110 | 111 | Provenance: 112 | needs: [Gather] 113 | permissions: 114 | actions: read 115 | id-token: write 116 | contents: write 117 | # Can't pin with hash due to how this workflow works. 118 | uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.1.0 119 | with: 120 | base64-subjects: ${{ needs.Gather.outputs.hash }} 121 | 122 | Release: 123 | runs-on: ubuntu-latest 124 | needs: [Provenance] 125 | permissions: 126 | contents: write 127 | if: startsWith(github.ref, 'refs/tags/') 128 | steps: 129 | - name: Checkout 130 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 131 | - name: Fetch Wheels 132 | uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 133 | with: 134 | pattern: "*" 135 | path: dist 136 | merge-multiple: true 137 | - name: Create Release 138 | uses: ncipollo/release-action@v1 139 | with: 140 | artifacts: "dist/*" 141 | token: ${{ github.token }} 142 | draft: false 143 | generateReleaseNotes: true 144 | 145 | Upload: 146 | needs: [Release] 147 | runs-on: ubuntu-latest 148 | if: startsWith(github.ref, 'refs/tags/') 149 | permissions: 150 | id-token: write 151 | steps: 152 | - name: Fetch Wheels 153 | uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 154 | with: 155 | pattern: "wheels-*" 156 | path: dist 157 | merge-multiple: true 158 | - name: Upload to PYPI 159 | uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc 160 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | workflow_dispatch: 10 | inputs: {} 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} 13 | cancel-in-progress: true 14 | jobs: 15 | Lint: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 20 | - name: Set up Python 21 | uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 22 | with: 23 | python-version: "3.13" 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install -r requirements-dev.txt 28 | - name: Black 29 | run: | 30 | black --check tfparse tests 31 | - name: Flake8 32 | run: | 33 | flake8 tfparse tests 34 | LeftTests: 35 | needs: Tests 36 | runs-on: ubuntu-latest 37 | steps: 38 | - name: Checkout 39 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 40 | 41 | - name: CheckoutLeft 42 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 43 | 44 | with: 45 | repository: cloud-custodian/cloud-custodian 46 | path: custodian 47 | 48 | - name: Set up Python 49 | uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 50 | with: 51 | python-version: 3.12 52 | 53 | - name: Set up Terraform 54 | uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd 55 | with: 56 | terraform_wrapper: false 57 | 58 | - name: Set up Go 59 | uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b 60 | with: 61 | go-version: "1.21.5" 62 | cache: true 63 | cache-dependency-path: "gotfparse/go.sum" 64 | 65 | - name: Set up Poetry 66 | shell: bash 67 | run: "pipx install poetry==1.5.1" 68 | 69 | - name: Setup virtualenv 70 | shell: bash 71 | run: "python -m venv .venv" 72 | 73 | - name: Activate virtualenv 74 | run: | 75 | echo "$PWD/.venv/bin" >> $GITHUB_PATH 76 | echo "VIRTUAL_ENV=$PWD/.venv" >> $GITHUB_ENV 77 | 78 | - name: Install c7n-left 79 | shell: bash 80 | working-directory: custodian/tools/c7n_left 81 | run: poetry install 82 | 83 | - name: Install dependencies 84 | run: | 85 | python -m pip install --upgrade pip 86 | pip install -r requirements-dev.txt 87 | 88 | - name: Install package 89 | run: | 90 | pip install -e . 91 | 92 | - name: Run c7n-left tests 93 | working-directory: custodian 94 | run: pytest tools/c7n_left/tests 95 | 96 | Tests: 97 | needs: Lint 98 | runs-on: ${{ matrix.runner }} 99 | strategy: 100 | matrix: 101 | python-version: ["3.10", "3.11", "3.12", "3.13"] 102 | runner: ["ubuntu-latest", "windows-latest", "macos-latest"] 103 | exclude: 104 | # just conserving runners by excluding older versions 105 | - runner: macos-latest 106 | python-version: 3.10 107 | - runner: windows-latest 108 | python-version: 3.10 109 | steps: 110 | - name: Checkout 111 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 112 | - name: Set up Python ${{ matrix.python-version }} 113 | uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 114 | with: 115 | python-version: ${{ matrix.python-version }} 116 | 117 | - name: Set up Terraform 118 | uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd 119 | with: 120 | terraform_wrapper: false 121 | 122 | - name: Set up Go 123 | uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b 124 | with: 125 | go-version: "1.21.5" 126 | cache: true 127 | cache-dependency-path: "gotfparse/go.sum" 128 | - name: Install dependencies 129 | run: | 130 | python -m pip install --upgrade pip 131 | pip install -r requirements-dev.txt 132 | - name: Install package 133 | run: | 134 | pip install -e . 135 | - name: Test with pytest for Python ${{ matrix.python-version }} 136 | run: | 137 | pytest tests 138 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .eggs/* 2 | dist/* 3 | build/* 4 | tfparse.egg-info 5 | tfparse/_tfparse.py 6 | venv/** 7 | .venv/** 8 | .tfcache 9 | *.so 10 | .python-version 11 | __pycache__ 12 | poetry.lock 13 | tfparse.cpython* 14 | .vscode 15 | 16 | .terraform 17 | .tf_providers 18 | .terraform.lock.hcl 19 | output.json 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2015-2017 Capital One Services, LLC 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # What 3 | 4 | A python extension for parsing and evaluating terraform using defsec. 5 | 6 | While terraform uses HCL as its configuration format, it requires numerous 7 | forms of variable interpolation, function and expression evaluation, which 8 | is beyond the typical usage of an hcl parser. To achieve compatibility 9 | with the myriad real world usages of terraform, this library uses the 10 | canonical implementation from terraform, along with the interpolation and evaluation 11 | from defsec to offer a high level interface to parsing terraform modules. 12 | 13 | # Installation 14 | 15 | ``` 16 | pip install tfparse 17 | ``` 18 | 19 | We currently distribute binaries for MacOS (x86_64, arm64) and Linux (x86_64, aarch64) and Windows. 20 | 21 | Note on Windows we currently don't free memory allocated on parse results. 22 | 23 | # Usage 24 | 25 | A terraform module root, with `terraform init` having been performed to resolve module references. 26 | 27 | ``` 28 | from tfparse import load_from_path 29 | parsed = load_from_path('path_to_terraform_root') 30 | print(parsed.keys()) 31 | ``` 32 | 33 | # Developing 34 | 35 | - requires Go >= 1.18 36 | - requires Python >= 3.10 37 | 38 | ## Installing from source 39 | 40 | Installing will build the module and install the local copy of tfparse in to the current Python environment. 41 | 42 | ```shell 43 | > pip install -e . 44 | > python 45 | >>> from tfparse import load_from_path 46 | >>> parsed = load_from_path('') 47 | >>> print(parsed.keys()) 48 | ``` 49 | 50 | ## Building from source 51 | 52 | Building will produce a wheel and a source artifact for distribution or upload to package repositories. 53 | 54 | ```shell 55 | python setup.py bdist_wheel 56 | ls -l dist/ 57 | ``` 58 | 59 | ## Running the tests 60 | 61 | This project uses pytest 62 | 63 | ```shell 64 | pytest 65 | ``` 66 | 67 | ## Testing CI Builds for cross compiling 68 | You can test our cross compiling CI/CD builds by running the following: 69 | 70 | ``` 71 | CIBW_BUILD=cp310* cibuildwheel --platform macos --archs x86_64 72 | ``` 73 | This will try to build an intel wheel on python3.10 74 | 75 | 76 | # Credits 77 | 78 | aquasecurity/defsec - golang module for parsing and evaluating terraform hcl 79 | 80 | Scalr/pygohcl - python bindings for terraform hcl via golang extension 81 | -------------------------------------------------------------------------------- /ci-setup-golang.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | OS=$(uname -s) 4 | ARCH=$(uname -m) 5 | GOVER="1.21.5" 6 | case $ARCH in 7 | x86_64) ARCH="amd64" ;; 8 | aarch64) ARCH="arm64" ;; 9 | esac 10 | case $OS in 11 | Darwin) OS="darwin" ;; 12 | Linux) OS="linux" ;; 13 | esac 14 | 15 | curl "https://storage.googleapis.com/golang/go${GOVER}.${OS}-${ARCH}.tar.gz" --silent --location | tar -xz 16 | 17 | export PATH="$(pwd)/go/bin:$PATH" 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /gotfparse/README.md: -------------------------------------------------------------------------------- 1 | # gotfparse 2 | 3 | `gotfparse` is a Go library that wraps the [defsec][defsec_repo] parser. This is done to provide an exported function to Python that takes advantage of the capabilities and speed of the defsec HCL parser. 4 | 5 | # Developing 6 | 7 | go mod tidy 8 | 9 | You can use the `tftest` helper command to easily iterate on terraform and preview the JSON output that the `gotfparse` library produces. 10 | 11 | go run cmd/tftest/main.go > output.json 12 | 13 | You can use the `tfdump` helper command to reveal the structure that `aquasecurity/defsec` creates to assist with identifying anomalies. It exports the original structure, along with interesting attributes, in json format. 14 | 15 | go run cmd/tfdump/main.go > output.json 16 | 17 | ## Tips 18 | 19 | When using a modern IDE like Visual Studio Code or Goland, open the `gotfparse` folder as the root of the workspace to ensure all of the Go tooling works as expected. 20 | 21 | 22 | [defsec_repo]: https://github.com/aquasecurity/defsec 23 | -------------------------------------------------------------------------------- /gotfparse/cmd/tfdump/main.go: -------------------------------------------------------------------------------- 1 | // Copyright The Cloud Custodian Authors. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "encoding/json" 9 | "fmt" 10 | "log" 11 | "os" 12 | "path/filepath" 13 | 14 | "github.com/aquasecurity/trivy/pkg/iac/scanners/terraform/parser" 15 | "github.com/aquasecurity/trivy/pkg/iac/terraform" 16 | "github.com/zclconf/go-cty/cty" 17 | ) 18 | 19 | func main() { 20 | if len(os.Args) < 2 || len(os.Args) > 2 { 21 | executable := filepath.Base(os.Args[0]) 22 | log.Fatalf("usage: %s PATH", executable) 23 | } 24 | 25 | var ( 26 | err error 27 | ctx = context.TODO() 28 | ) 29 | 30 | path := os.Args[1] 31 | fs := os.DirFS(path) 32 | p := parser.New(fs, "") 33 | 34 | err = p.ParseFS(ctx, ".") 35 | check(err) 36 | 37 | modules, err := p.EvaluateAll(ctx) 38 | check(err) 39 | 40 | objects, err := dumpJson(modules) 41 | check(err) 42 | 43 | data, err := json.MarshalIndent(objects, "", " ") 44 | check(err) 45 | 46 | fmt.Println(string(data)) 47 | } 48 | 49 | func check(err error) { 50 | if err != nil { 51 | panic(err) 52 | } 53 | } 54 | 55 | func dumpJson(modules terraform.Modules) ([]map[string]interface{}, error) { 56 | output := make([]map[string]interface{}, 0) 57 | 58 | for _, m := range modules { 59 | output = append(output, dumpModule(m)...) 60 | } 61 | 62 | return output, nil 63 | } 64 | 65 | func dumpModule(m *terraform.Module) []map[string]interface{} { 66 | output := make([]map[string]interface{}, 0) 67 | 68 | for _, b := range m.GetBlocks() { 69 | output = append(output, dumpBlock(m, b)) 70 | } 71 | return output 72 | } 73 | 74 | func dumpBlock(m *terraform.Module, b *terraform.Block) map[string]interface{} { 75 | object := make(map[string]interface{}) 76 | object["__full_name__"] = b.FullName() 77 | object["__id__"] = b.ID() 78 | object["__is_expanded"] = b.IsExpanded() 79 | object["__is_empty__"] = b.IsEmpty() 80 | object["__in_module__"] = b.InModule() 81 | object["__is_nil__"] = b.IsNil() 82 | object["__is_not_nil__"] = b.IsNotNil() 83 | object["__label__"] = b.Label() 84 | object["__labels__"] = b.Labels() 85 | object["__local_name__"] = b.LocalName() 86 | object["__module_name__"] = b.ModuleKey() 87 | object["__name_label__"] = b.NameLabel() 88 | object["__type__"] = b.Type() 89 | object["__type_label__"] = b.TypeLabel() 90 | object["__unique_name__"] = b.UniqueName() 91 | 92 | r := b.GetMetadata().Range() 93 | 94 | object["__ref__"] = map[string]interface{}{ 95 | "start": r.GetStartLine(), 96 | "end": r.GetEndLine(), 97 | "fname": r.GetFilename(), 98 | "local": r.GetLocalFilename(), 99 | "prefix": r.GetSourcePrefix(), 100 | } 101 | 102 | for _, attr := range b.GetAttributes() { 103 | object[attr.Name()] = dumpAttribute(m, b, attr) 104 | } 105 | 106 | children := make([]interface{}, 0) 107 | for _, sub := range b.AllBlocks() { 108 | children = append(children, dumpBlock(m, sub)) 109 | } 110 | if len(children) > 0 { 111 | object["__children__"] = children 112 | } 113 | 114 | return object 115 | } 116 | 117 | func dumpAttribute(m *terraform.Module, parent *terraform.Block, attr *terraform.Attribute) interface{} { 118 | t := attr.Type() 119 | 120 | if attr.IsDataBlockReference() { 121 | ref, err := m.GetReferencedBlock(attr, parent) 122 | if err != nil { 123 | return map[string]interface{}{ 124 | "__missing_ref__": attr.GetMetadata().Reference(), 125 | } 126 | } 127 | 128 | return map[string]interface{}{"__ref_id__": ref.ID()} 129 | } 130 | 131 | if t.IsTupleType() || t.IsListType() { 132 | var result []interface{} 133 | for _, v := range attr.Value().AsValueSlice() { 134 | result = append(result, getRawValue(v)) 135 | } 136 | return result 137 | } 138 | return attr.GetRawValue() 139 | } 140 | 141 | func getRawValue(a cty.Value) interface{} { 142 | if !a.IsKnown() { 143 | return "<>" 144 | } 145 | 146 | if a.IsNull() { 147 | return nil 148 | } 149 | 150 | switch typ := a.Type(); typ { 151 | case cty.String: 152 | return a.AsString() 153 | case cty.Bool: 154 | return a.True() 155 | case cty.Number: 156 | float, _ := a.AsBigFloat().Float64() 157 | return float 158 | default: 159 | switch { 160 | case typ.IsTupleType(), typ.IsListType(): 161 | values := a.AsValueSlice() 162 | 163 | var result []interface{} 164 | for _, v := range values { 165 | result = append(result, getRawValue(v)) 166 | } 167 | return result 168 | } 169 | } 170 | return nil 171 | } 172 | -------------------------------------------------------------------------------- /gotfparse/cmd/tfparse/main.go: -------------------------------------------------------------------------------- 1 | // Copyright The Cloud Custodian Authors. 2 | // SPDX-License-Identifier: Apache-2.0 3 | package main 4 | 5 | // typedef struct { 6 | // char *json; 7 | // char *err; 8 | // } parseResponse; 9 | import "C" 10 | import ( 11 | "fmt" 12 | "unsafe" 13 | 14 | "github.com/cloud-custodian/tfparse/gotfparse/pkg/converter" 15 | ) 16 | 17 | //export Parse 18 | func Parse(a *C.char, stopHCL C.int, debug C.int, allowDownloads C.int, workspaceName *C.char, num_vars_files C.int, vars_files **C.char) (resp C.parseResponse) { 19 | input := C.GoString(a) 20 | 21 | options := []converter.TerraformConverterOption{} 22 | if stopHCL != 0 { 23 | options = append(options, converter.WithStopOnHCLError()) 24 | } 25 | 26 | if debug != 0 { 27 | options = append(options, converter.WithDebug()) 28 | } 29 | 30 | if allowDownloads != 0 { 31 | options = append(options, converter.WithAllowDownloads(true)) 32 | } else { 33 | options = append(options, converter.WithAllowDownloads(false)) 34 | } 35 | 36 | options = append(options, converter.WithWorkspaceName(C.GoString(workspaceName))) 37 | 38 | var varFiles []string 39 | for _, v := range unsafe.Slice(vars_files, num_vars_files) { 40 | varFiles = append(varFiles, C.GoString(v)) 41 | } 42 | if len(varFiles) != 0 { 43 | options = append(options, converter.WithTFVarsPaths(varFiles...)) 44 | } 45 | 46 | tfd, err := converter.NewTerraformConverter(input, options...) 47 | if err != nil { 48 | return C.parseResponse{nil, C.CString(fmt.Sprintf("unable to create TerraformConverter: %s", err))} 49 | } 50 | j, err := tfd.VisitJSON().MarshalJSON() 51 | if err != nil { 52 | return C.parseResponse{nil, C.CString(fmt.Sprintf("cannot generate JSON from path: %s", err))} 53 | } 54 | 55 | resp = C.parseResponse{C.CString(string(j)), nil} 56 | return resp 57 | } 58 | 59 | func main() {} 60 | -------------------------------------------------------------------------------- /gotfparse/cmd/tftest/README.md: -------------------------------------------------------------------------------- 1 | # tftest 2 | 3 | `tftest` is used when developing to allow for rapid iteration of the gotfparse library. 4 | 5 | go run cmd/tftest/main.go > output.json -------------------------------------------------------------------------------- /gotfparse/cmd/tftest/main.go: -------------------------------------------------------------------------------- 1 | // Copyright The Cloud Custodian Authors. 2 | // SPDX-License-Identifier: Apache-2.0 3 | package main 4 | 5 | import ( 6 | "encoding/json" 7 | "fmt" 8 | "log" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/cloud-custodian/tfparse/gotfparse/pkg/converter" 14 | ) 15 | 16 | func main() { 17 | if len(os.Args) < 2 { 18 | executable := filepath.Base(os.Args[0]) 19 | log.Fatalf("usage: %s PATH [--debug]", executable) 20 | } 21 | 22 | // Check arguments for debug flag 23 | var path string 24 | debug := false 25 | 26 | for _, arg := range os.Args[1:] { 27 | if arg == "--debug" { 28 | debug = true 29 | } else if !strings.HasPrefix(arg, "--") { 30 | path = arg 31 | } 32 | } 33 | 34 | if path == "" { 35 | executable := filepath.Base(os.Args[0]) 36 | log.Fatalf("usage: %s PATH [--debug]", executable) 37 | } 38 | 39 | // Create converter with options 40 | opts := []converter.TerraformConverterOption{} 41 | if debug { 42 | opts = append(opts, converter.WithDebug()) 43 | } 44 | 45 | tfd, err := converter.NewTerraformConverter(path, opts...) 46 | checkError(err) 47 | 48 | data := tfd.VisitJSON().Data() 49 | 50 | j, err := json.MarshalIndent(data, "", "\t") 51 | checkError(err) 52 | 53 | fmt.Print(string(j)) 54 | } 55 | 56 | func checkError(err error) { 57 | if err == nil { 58 | return 59 | } 60 | 61 | panic(err) 62 | } 63 | -------------------------------------------------------------------------------- /gotfparse/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/cloud-custodian/tfparse/gotfparse 2 | 3 | go 1.24.2 4 | 5 | toolchain go1.24.3 6 | 7 | require ( 8 | github.com/Jeffail/gabs/v2 v2.7.0 9 | github.com/aquasecurity/trivy v0.62.1 10 | github.com/hashicorp/hcl/v2 v2.23.0 11 | github.com/zclconf/go-cty v1.16.3 12 | ) 13 | 14 | require ( 15 | cel.dev/expr v0.24.0 // indirect 16 | cloud.google.com/go v0.121.2 // indirect 17 | cloud.google.com/go/auth v0.16.1 // indirect 18 | cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect 19 | cloud.google.com/go/compute/metadata v0.7.0 // indirect 20 | cloud.google.com/go/iam v1.5.2 // indirect 21 | cloud.google.com/go/monitoring v1.24.2 // indirect 22 | cloud.google.com/go/storage v1.54.0 // indirect 23 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect 24 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect 25 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect 26 | github.com/agext/levenshtein v1.2.3 // indirect 27 | github.com/apparentlymart/go-cidr v1.1.0 // indirect 28 | github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect 29 | github.com/aquasecurity/go-version v0.0.1 // indirect 30 | github.com/aws/aws-sdk-go v1.55.7 // indirect 31 | github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect 32 | github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect 33 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 34 | github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect 35 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect 36 | github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect 37 | github.com/fatih/color v1.18.0 // indirect 38 | github.com/felixge/httpsnoop v1.0.4 // indirect 39 | github.com/go-jose/go-jose/v4 v4.1.0 // indirect 40 | github.com/go-logr/logr v1.4.2 // indirect 41 | github.com/go-logr/stdr v1.2.2 // indirect 42 | github.com/google/s2a-go v0.1.9 // indirect 43 | github.com/google/uuid v1.6.0 // indirect 44 | github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect 45 | github.com/googleapis/gax-go/v2 v2.14.2 // indirect 46 | github.com/hashicorp/errwrap v1.1.0 // indirect 47 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 48 | github.com/hashicorp/go-getter v1.7.8 // indirect 49 | github.com/hashicorp/go-multierror v1.1.1 // indirect 50 | github.com/hashicorp/go-safetemp v1.0.0 // indirect 51 | github.com/hashicorp/go-uuid v1.0.3 // indirect 52 | github.com/hashicorp/go-version v1.7.0 // indirect 53 | github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect 54 | github.com/klauspost/compress v1.18.0 // indirect 55 | github.com/mattn/go-colorable v0.1.14 // indirect 56 | github.com/mattn/go-isatty v0.0.20 // indirect 57 | github.com/mitchellh/go-homedir v1.1.0 // indirect 58 | github.com/mitchellh/go-testing-interface v1.14.1 // indirect 59 | github.com/mitchellh/go-wordwrap v1.0.1 // indirect 60 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect 61 | github.com/samber/lo v1.50.0 // indirect 62 | github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect 63 | github.com/ulikunitz/xz v0.5.12 // indirect 64 | github.com/zclconf/go-cty-yaml v1.1.0 // indirect 65 | github.com/zeebo/errs v1.4.0 // indirect 66 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 67 | go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect 68 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect 69 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect 70 | go.opentelemetry.io/otel v1.36.0 // indirect 71 | go.opentelemetry.io/otel/metric v1.36.0 // indirect 72 | go.opentelemetry.io/otel/sdk v1.36.0 // indirect 73 | go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect 74 | go.opentelemetry.io/otel/trace v1.36.0 // indirect 75 | golang.org/x/crypto v0.38.0 // indirect 76 | golang.org/x/mod v0.24.0 // indirect 77 | golang.org/x/net v0.40.0 // indirect 78 | golang.org/x/oauth2 v0.30.0 // indirect 79 | golang.org/x/sync v0.14.0 // indirect 80 | golang.org/x/sys v0.33.0 // indirect 81 | golang.org/x/text v0.25.0 // indirect 82 | golang.org/x/time v0.11.0 // indirect 83 | golang.org/x/tools v0.33.0 // indirect 84 | golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect 85 | google.golang.org/api v0.234.0 // indirect 86 | google.golang.org/genproto v0.0.0-20250519155744-55703ea1f237 // indirect 87 | google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 // indirect 88 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect 89 | google.golang.org/grpc v1.72.2 // indirect 90 | google.golang.org/protobuf v1.36.6 // indirect 91 | k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979 // indirect 92 | ) 93 | 94 | replace github.com/aquasecurity/trivy => github.com/cloud-custodian/trivy v0.0.0-20250528004150-d6b7e2605cc3 95 | -------------------------------------------------------------------------------- /gotfparse/pkg/converter/converter.go: -------------------------------------------------------------------------------- 1 | // Copyright The Cloud Custodian Authors. 2 | // SPDX-License-Identifier: Apache-2.0 3 | package converter 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "log/slog" 9 | "os" 10 | "slices" 11 | "strconv" 12 | "strings" 13 | 14 | "github.com/Jeffail/gabs/v2" 15 | "github.com/aquasecurity/trivy/pkg/iac/scanners/terraform/parser" 16 | "github.com/aquasecurity/trivy/pkg/iac/terraform" 17 | "github.com/hashicorp/hcl/v2" 18 | "github.com/hashicorp/hcl/v2/hclsyntax" 19 | "github.com/zclconf/go-cty/cty" 20 | "github.com/zclconf/go-cty/cty/function" 21 | ) 22 | 23 | // Default to INFO level 24 | var logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ 25 | Level: slog.LevelInfo, 26 | })) 27 | 28 | // SetLogLevel sets the logging level 29 | func SetLogLevel(level slog.Level) { 30 | handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ 31 | Level: level, 32 | }) 33 | logger = slog.New(handler) 34 | } 35 | 36 | type stringSet map[string]bool 37 | 38 | func (s *stringSet) Add(str string) { 39 | if !(*s)[str] { 40 | (*s)[str] = true 41 | } 42 | } 43 | 44 | func (s stringSet) Entries() []string { 45 | entries := make([]string, len(s)) 46 | i := 0 47 | for entry, _ := range s { 48 | entries[i] = entry 49 | i++ 50 | } 51 | return entries 52 | } 53 | 54 | type blockReferences struct { 55 | refs []string 56 | // block metadata 57 | meta *map[string]any 58 | } 59 | 60 | type referenceTracker struct { 61 | // track blocks by their string reference 62 | blocksByReference map[string]*terraform.Block 63 | // track all processed blocks that might have references 64 | blocksWithReferences []*blockReferences 65 | } 66 | 67 | func (r *referenceTracker) AddBlock(b *terraform.Block) { 68 | r.blocksByReference[b.FullName()] = b 69 | } 70 | 71 | func (r *referenceTracker) AddBlockReferences(refs []string, blockMeta *map[string]any) { 72 | slices.Sort(refs) // for consistent ordering in block metadata 73 | r.blocksWithReferences = append(r.blocksWithReferences, &blockReferences{refs: refs, meta: blockMeta}) 74 | } 75 | 76 | // ProcessBlocksReferences includes a "references" entry in blocks metadata if 77 | // they have references to other blocks. This must be called once all 78 | // references have been collected for all blocks. 79 | func (r *referenceTracker) ProcessBlocksReferences() { 80 | for _, blockRef := range r.blocksWithReferences { 81 | refsMeta := [](map[string]any){} 82 | for _, ref := range blockRef.refs { 83 | if block, ok := r.blocksByReference[ref]; ok { 84 | meta := map[string]any{ 85 | "id": block.ID(), 86 | "label": block.TypeLabel(), 87 | "name": block.NameLabel(), 88 | } 89 | refsMeta = append(refsMeta, meta) 90 | } 91 | } 92 | if len(refsMeta) > 0 { 93 | (*blockRef.meta)["references"] = refsMeta 94 | } 95 | } 96 | } 97 | 98 | func newReferenceTracker() referenceTracker { 99 | return referenceTracker{ 100 | blocksByReference: make(map[string]*terraform.Block), 101 | blocksWithReferences: []*blockReferences{}, 102 | } 103 | } 104 | 105 | type terraformConverter struct { 106 | filePath string 107 | modules terraform.Modules 108 | debug bool 109 | stopOnError bool 110 | parserOptions []parser.Option 111 | referenceTracker referenceTracker 112 | } 113 | 114 | // VisitJSON visits each of the Terraform JSON blocks that the Terraform converter 115 | // has stored in memory and extracts addiontal metadata from the underlying defsec data 116 | // structure and embeds the metadata directly in to the JSON data. 117 | func (t *terraformConverter) VisitJSON() *gabs.Container { 118 | jsonOut := gabs.New() 119 | 120 | for _, m := range t.modules { 121 | t.visitModule(m, jsonOut) 122 | } 123 | 124 | // Now that all blocks have been processed, fill metadata about related 125 | // blocks for labels collected during visiting 126 | t.referenceTracker.ProcessBlocksReferences() 127 | 128 | return jsonOut 129 | } 130 | 131 | // visitModule takes a module and walks each of the blocks underneath it. 132 | func (t *terraformConverter) visitModule(m *terraform.Module, out *gabs.Container) { 133 | path := t.getModulePath(m) 134 | 135 | for _, b := range m.GetBlocks() { 136 | t.visitBlock(b, path, out) 137 | } 138 | } 139 | 140 | // visitBlock takes a block, and either builds a json model of the resource or ignores it. 141 | func (t *terraformConverter) visitBlock(b *terraform.Block, parentPath string, jsonOut *gabs.Container) { 142 | t.referenceTracker.AddBlock(b) 143 | 144 | switch b.Type() { 145 | // These blocks don't have to conform to policies, and they don't have 146 | //children that should have policies applied to them, so we ignore them. 147 | case "data", "locals", "output", "provider", "terraform", "variable", "module", "moved", "resource": 148 | json := t.buildBlock(b) 149 | meta := json["__tfmeta"].(map[string]interface{}) 150 | 151 | arrayKey := t.getPath(b, parentPath) 152 | 153 | meta["path"] = arrayKey 154 | 155 | var key string 156 | switch b.Type() { 157 | case "data", "resource": 158 | key = b.TypeLabel() 159 | meta["type"] = b.Type() 160 | 161 | default: 162 | key = b.Type() 163 | } 164 | 165 | jsonOut.ArrayAppendP(json, key) 166 | default: 167 | logger.Info("unknown block type", "type", b.Type()) 168 | } 169 | } 170 | 171 | // getList gets a slice from 172 | func getList(obj map[string][]interface{}, key string) []interface{} { 173 | value, ok := obj[key] 174 | if !ok { 175 | value = make([]interface{}, 0) 176 | obj[key] = value 177 | } 178 | return value 179 | } 180 | 181 | type add func(string, interface{}) 182 | type dump func() map[string]interface{} 183 | 184 | // newBlockCollector creates a few closures to help flatten 185 | // lists that are actually singletons. 186 | // Note: This doesn't guarantee that they're _supposed_ to be singeltons, only 187 | // that there is only a single item in the list as rendered. 188 | func newBlockCollector() (add, dump) { 189 | collection := make(map[string][]interface{}) 190 | 191 | add := func(key string, value interface{}) { 192 | list := getList(collection, key) 193 | collection[key] = append(list, value) 194 | } 195 | 196 | dump := func() map[string]interface{} { 197 | results := make(map[string]interface{}) 198 | for key, items := range collection { 199 | if len(items) == 1 { 200 | results[key] = items[0] 201 | } else { 202 | results[key] = items 203 | } 204 | } 205 | return results 206 | } 207 | 208 | return add, dump 209 | } 210 | 211 | // buildBlock converts a terraform.Block's attributes and children to a json map. 212 | func (t *terraformConverter) buildBlock(b *terraform.Block) map[string]interface{} { 213 | obj := make(map[string]interface{}) 214 | 215 | add, dump := newBlockCollector() 216 | for _, child := range getChildBlocks(b) { 217 | key := child.Type() 218 | add(key, t.buildBlock(child)) 219 | } 220 | grouped := dump() 221 | for key, result := range grouped { 222 | obj[key] = result 223 | } 224 | 225 | allRefs := stringSet{} 226 | for _, a := range b.GetAttributes() { 227 | attrName := a.Name() 228 | if b.Type() == "variable" && attrName == "type" { 229 | // for variable type, the plain value is nil (unless the type has 230 | // been provided in quotes), look at the variable type instead 231 | var_type, _, _ := a.DecodeVarType() 232 | obj[attrName] = var_type.FriendlyName() 233 | } else { 234 | obj[attrName] = t.getAttributeValue(a) 235 | } 236 | 237 | for _, ref := range a.AllReferences() { 238 | allRefs.Add(ref.String()) 239 | } 240 | } 241 | 242 | if id := b.ID(); id != "" { 243 | obj["id"] = id 244 | } 245 | 246 | r := b.GetMetadata().Range() 247 | meta := map[string]interface{}{ 248 | "filename": r.GetLocalFilename(), 249 | "line_start": r.GetStartLine(), 250 | "line_end": r.GetEndLine(), 251 | } 252 | 253 | if refs := allRefs.Entries(); len(refs) > 0 { 254 | t.referenceTracker.AddBlockReferences(refs, &meta) 255 | } 256 | if tl := b.TypeLabel(); tl != "" { 257 | meta["label"] = tl 258 | } 259 | obj["__tfmeta"] = meta 260 | return obj 261 | } 262 | 263 | // getAttributeValue returns the value for the attribute 264 | func (t *terraformConverter) getAttributeValue(a *terraform.Attribute) any { 265 | // First try using the parsed value directly 266 | val := a.Value() 267 | 268 | // Only attempt to handle functions manually if the value is null or not known 269 | // This ensures we don't interfere with functions that have been successfully resolved 270 | if val.IsNull() || !val.IsKnown() { 271 | // Check if it's a function call that might have failed due to unresolvable variables 272 | hclAttr := getPrivateValue(a, "hclAttribute").(*hcl.Attribute) 273 | 274 | // Try different expression types in order of precedence 275 | if templateExpr, isTemplate := hclAttr.Expr.(*hclsyntax.TemplateExpr); isTemplate { 276 | return t.handleTemplateExpression(a, templateExpr) 277 | } 278 | 279 | if traversalExpr, isTraversal := hclAttr.Expr.(*hclsyntax.ScopeTraversalExpr); isTraversal { 280 | return t.handleScopeTraversal(traversalExpr) 281 | } 282 | 283 | if funcExpr, isFuncCall := hclAttr.Expr.(*hclsyntax.FunctionCallExpr); isFuncCall { 284 | return t.handleFunctionCall(funcExpr) 285 | } 286 | } 287 | 288 | // Try to convert the value to a native type 289 | if raw, ok := convertCtyToNativeValue(val); ok { 290 | return raw 291 | } 292 | 293 | // If we get this far, just return the raw value 294 | return a.GetRawValue() 295 | } 296 | 297 | // handleTemplateExpression processes string interpolation expressions 298 | func (t *terraformConverter) handleTemplateExpression(a *terraform.Attribute, templateExpr *hclsyntax.TemplateExpr) any { 299 | // For string interpolation that couldn't be fully resolved, 300 | // try to reconstruct the original template 301 | rawValue := a.GetRawValue() 302 | if rawValue != nil && rawValue != "" && rawValue != "null" { 303 | // If we have a non-empty raw value, use it 304 | return rawValue 305 | } 306 | 307 | // If raw value isn't helpful, try to reconstruct the template from its parts 308 | reconstructed := reconstructTemplate(templateExpr) 309 | if reconstructed != "" { 310 | return reconstructed 311 | } 312 | 313 | // Fallback to template placeholder 314 | return fmt.Sprintf("