├── .github └── workflows │ └── cfn-guard.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MultiAccountApplication ├── .gitignore ├── ccft-org-read-role │ ├── ccft-role │ │ └── ccft-read-role.yaml │ └── manifest.yaml ├── lambda_functions │ ├── __init__.py │ ├── check_first_invocation │ │ ├── __init__.py │ │ └── app.py │ ├── create_alter_athena_view │ │ ├── __init__.py │ │ ├── app.py │ │ ├── create_aggregate_view.sql │ │ ├── create_forecast_view.sql │ │ ├── create_table.ddl │ │ ├── create_view.sql │ │ └── util.py │ ├── extract_carbon_emissions │ │ ├── __init__.py │ │ ├── app.py │ │ ├── ccft_access.py │ │ ├── requirements-dev.txt │ │ └── requirements.txt │ └── get_account_ids │ │ └── app.py ├── statemachine │ └── extract_carbon_emissions.asl.json ├── template.yaml └── tests │ ├── __init__.py │ ├── requirements.txt │ └── unit │ ├── __init__.py │ ├── test.sql │ ├── test_create_alter_athena_view.py │ └── test_get_account_ids.py ├── README.md ├── cfn-guard-rules └── wa-Security-Pillar.guard ├── package.json └── static ├── athena_table_view.png ├── ccft-api-explanation.png ├── ccft-sam_architecture.jpg └── stepfunctions_graph.svg /.github/workflows/cfn-guard.yml: -------------------------------------------------------------------------------- 1 | name: CloudFormation Guard Validate 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | guard: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | # only required when create-review is true (default) 11 | pull-requests: write 12 | name: CloudFormation Guard validate 13 | steps: 14 | - name: CloudFormation Guard validate 15 | uses: aws-cloudformation/cloudformation-guard@action-v0.0.4 16 | with: 17 | rules: './cfn-guard-rules/' 18 | data: './MultiAccountApplication/' 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode 3 | 4 | ### Linux ### 5 | *~ 6 | 7 | # temporary files which can be created if a process still has a handle open of a deleted file 8 | .fuse_hidden* 9 | 10 | # KDE directory preferences 11 | .directory 12 | 13 | # Linux trash folder which might appear on any partition or disk 14 | .Trash-* 15 | 16 | # .nfs files are created when an open file is removed but is still being accessed 17 | .nfs* 18 | 19 | ### OSX ### 20 | *.DS_Store 21 | .AppleDouble 22 | .LSOverride 23 | 24 | # Icon must end with two \r 25 | Icon 26 | 27 | # Thumbnails 28 | ._* 29 | 30 | # Files that might appear in the root of a volume 31 | .DocumentRevisions-V100 32 | .fseventsd 33 | .Spotlight-V100 34 | .TemporaryItems 35 | .Trashes 36 | .VolumeIcon.icns 37 | .com.apple.timemachine.donotpresent 38 | 39 | # Directories potentially created on remote AFP share 40 | .AppleDB 41 | .AppleDesktop 42 | Network Trash Folder 43 | Temporary Items 44 | .apdisk 45 | 46 | ### PyCharm ### 47 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 48 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 49 | 50 | # User-specific stuff: 51 | .idea/**/workspace.xml 52 | .idea/**/tasks.xml 53 | .idea/dictionaries 54 | 55 | # Sensitive or high-churn files: 56 | .idea/**/dataSources/ 57 | .idea/**/dataSources.ids 58 | .idea/**/dataSources.xml 59 | .idea/**/dataSources.local.xml 60 | .idea/**/sqlDataSources.xml 61 | .idea/**/dynamic.xml 62 | .idea/**/uiDesigner.xml 63 | 64 | # Gradle: 65 | .idea/**/gradle.xml 66 | .idea/**/libraries 67 | 68 | # CMake 69 | cmake-build-debug/ 70 | 71 | # Mongo Explorer plugin: 72 | .idea/**/mongoSettings.xml 73 | 74 | ## File-based project format: 75 | *.iws 76 | 77 | ## Plugin-specific files: 78 | 79 | # IntelliJ 80 | /out/ 81 | 82 | # mpeltonen/sbt-idea plugin 83 | .idea_modules/ 84 | 85 | # JIRA plugin 86 | atlassian-ide-plugin.xml 87 | 88 | # Cursive Clojure plugin 89 | .idea/replstate.xml 90 | 91 | # Ruby plugin and RubyMine 92 | /.rakeTasks 93 | 94 | # Crashlytics plugin (for Android Studio and IntelliJ) 95 | com_crashlytics_export_strings.xml 96 | crashlytics.properties 97 | crashlytics-build.properties 98 | fabric.properties 99 | 100 | ### PyCharm Patch ### 101 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 102 | 103 | # *.iml 104 | # modules.xml 105 | # .idea/misc.xml 106 | # *.ipr 107 | 108 | # Sonarlint plugin 109 | .idea/sonarlint 110 | 111 | ### Python ### 112 | # Byte-compiled / optimized / DLL files 113 | __pycache__/ 114 | *.py[cod] 115 | *$py.class 116 | 117 | # C extensions 118 | *.so 119 | 120 | # Distribution / packaging 121 | .Python 122 | build/ 123 | develop-eggs/ 124 | dist/ 125 | downloads/ 126 | eggs/ 127 | .eggs/ 128 | lib/ 129 | lib64/ 130 | parts/ 131 | sdist/ 132 | var/ 133 | wheels/ 134 | *.egg-info/ 135 | .installed.cfg 136 | *.egg 137 | 138 | # PyInstaller 139 | # Usually these files are written by a python script from a template 140 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 141 | *.manifest 142 | *.spec 143 | 144 | # Installer logs 145 | pip-log.txt 146 | pip-delete-this-directory.txt 147 | 148 | # Unit test / coverage reports 149 | htmlcov/ 150 | .tox/ 151 | .coverage 152 | .coverage.* 153 | .cache 154 | .pytest_cache/ 155 | nosetests.xml 156 | coverage.xml 157 | *.cover 158 | .hypothesis/ 159 | 160 | # Translations 161 | *.mo 162 | *.pot 163 | 164 | # Flask stuff: 165 | instance/ 166 | .webassets-cache 167 | 168 | # Scrapy stuff: 169 | .scrapy 170 | 171 | # Sphinx documentation 172 | docs/_build/ 173 | 174 | # PyBuilder 175 | target/ 176 | 177 | # Jupyter Notebook 178 | .ipynb_checkpoints 179 | 180 | # pyenv 181 | .python-version 182 | 183 | # celery beat schedule file 184 | celerybeat-schedule.* 185 | 186 | # SageMath parsed files 187 | *.sage.py 188 | 189 | # Environments 190 | .env 191 | .venv 192 | env/ 193 | venv/ 194 | ENV/ 195 | env.bak/ 196 | venv.bak/ 197 | 198 | # Spyder project settings 199 | .spyderproject 200 | .spyproject 201 | 202 | # Rope project settings 203 | .ropeproject 204 | 205 | # mkdocs documentation 206 | /site 207 | 208 | # mypy 209 | .mypy_cache/ 210 | 211 | ### VisualStudioCode ### 212 | .vscode/* 213 | !.vscode/settings.json 214 | !.vscode/tasks.json 215 | !.vscode/launch.json 216 | !.vscode/extensions.json 217 | .history 218 | 219 | ### Windows ### 220 | # Windows thumbnail cache files 221 | Thumbs.db 222 | ehthumbs.db 223 | ehthumbs_vista.db 224 | 225 | # Folder config file 226 | Desktop.ini 227 | 228 | # Recycle Bin used on file shares 229 | $RECYCLE.BIN/ 230 | 231 | # Windows Installer files 232 | *.cab 233 | *.msi 234 | *.msm 235 | *.msp 236 | 237 | # Windows shortcuts 238 | *.lnk 239 | 240 | # Build folder 241 | 242 | */build/* 243 | .aws-sam/ 244 | 245 | # End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT No Attribution 2 | 3 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 13 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 15 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 16 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | -------------------------------------------------------------------------------- /MultiAccountApplication/.gitignore: -------------------------------------------------------------------------------- 1 | samconfig.toml 2 | -------------------------------------------------------------------------------- /MultiAccountApplication/ccft-org-read-role/ccft-role/ccft-read-role.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Description: > 3 | Creates a stack containing an IAM Role for reading CCFT data. 4 | 5 | Belongs to the SAM template to automate the monthly extraction of new CCFT data within a multi account structure (uksb-880c16twdr). 6 | Parameters: 7 | CCFTReadDataRole: 8 | Type: String 9 | Default: ccft-read-role 10 | Description: Role for reading CCFT data 11 | ManagementAccount: 12 | Description: Management Account ID number 13 | Type: String 14 | Resources: 15 | CCFTReadAccountDataRole: 16 | Type: 'AWS::IAM::Role' 17 | Metadata: 18 | guard: 19 | SuppressedRules: 20 | # we don't need to reuse the policy in other roles and inline it 21 | - IAM_NO_INLINE_POLICY_CHECK 22 | Properties: 23 | RoleName: 24 | !Ref CCFTReadDataRole 25 | Path: / 26 | AssumeRolePolicyDocument: 27 | Version: "2012-10-17" 28 | Statement: 29 | - Effect: Allow 30 | Principal: 31 | AWS: 32 | - !Sub 'arn:aws:iam::${ManagementAccount}:role/extract-emissions-lambda-role' 33 | Action: 34 | - 'sts:AssumeRole' 35 | Policies: 36 | - PolicyName: CCFT-Read-Data-Policy 37 | PolicyDocument: 38 | Version: "2012-10-17" 39 | Statement: 40 | - Sid: CCFTReadData 41 | Effect: Allow 42 | Resource: "*" 43 | Action: 44 | - 'sustainability:GetCarbonFootprintSummary' -------------------------------------------------------------------------------- /MultiAccountApplication/ccft-org-read-role/manifest.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | #Default region for deploying Custom Control Tower: Code Pipeline, Step functions, Lambda, SSM parameters, and StackSets 3 | region: us-east-1 4 | version: 2021-03-15 5 | 6 | resources: 7 | # ----------------------------------------------------------------------------- 8 | # Role for reading CCFT data in spoke accounts 9 | # ----------------------------------------------------------------------------- 10 | - name: CCFTSpokeAccountsReadDataRole 11 | resource_file: ccft-role/ccft-read-role.yaml 12 | parameters: 13 | - parameter_key: "CCFTReadDataRole" 14 | parameter_value: "ccft-read-role" 15 | - parameter_key: "ManagementAccount" 16 | parameter_value: "$[alfred_ssm_/org/account-id/management]" # or AccountID 17 | deploy_method: stack_set 18 | deployment_targets: 19 | accounts: 20 | - Management-Account-Name # Mgmt has to be targeted individually 21 | organizational_units: # List here all the OUs you wish to deploy to. Samples below. 22 | - Security 23 | - Infrastructure 24 | - NonProduction 25 | - Production 26 | regions: 27 | - us-east-1 -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/experimental-programmatic-access-ccft/d1c273542acee52adedfaf85a44d48d865929a95/MultiAccountApplication/lambda_functions/__init__.py -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/check_first_invocation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/experimental-programmatic-access-ccft/d1c273542acee52adedfaf85a44d48d865929a95/MultiAccountApplication/lambda_functions/check_first_invocation/__init__.py -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/check_first_invocation/app.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import os 3 | 4 | def lambda_handler(event, context): 5 | 6 | s3_bucket = os.environ['bucketName'] 7 | 8 | # Create an S3 client 9 | s3 = boto3.client('s3') 10 | 11 | # Retrieve the list of objects in the bucket 12 | response = s3.list_objects_v2(Bucket=s3_bucket) 13 | 14 | # Check if the bucket is empty 15 | if 'Contents' in response: 16 | is_empty = False 17 | else: 18 | is_empty = True 19 | 20 | return { 21 | 'statusCode': 200, 22 | 'isEmpty': is_empty 23 | } 24 | -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/create_alter_athena_view/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/experimental-programmatic-access-ccft/d1c273542acee52adedfaf85a44d48d865929a95/MultiAccountApplication/lambda_functions/create_alter_athena_view/__init__.py -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/create_alter_athena_view/app.py: -------------------------------------------------------------------------------- 1 | import os 2 | import util 3 | 4 | def lambda_handler(*_): 5 | 6 | workgroup_name = os.environ['workgroupName'] 7 | database_name = os.environ['glueDatabaseName'] 8 | 9 | placeholders = { 10 | "emissions_location": "s3://"+os.environ['emissionsBucketName']+"/", 11 | "database_name": database_name 12 | } 13 | 14 | util.run_query(f"CREATE DATABASE IF NOT EXISTS {database_name}", workgroup_name) 15 | 16 | util.run_query(util.load_query('create_table.ddl', placeholders), workgroup_name) 17 | 18 | util.run_query(util.load_query('create_view.sql', placeholders), workgroup_name) 19 | 20 | util.run_query(util.load_query('create_aggregate_view.sql', placeholders), workgroup_name) 21 | 22 | util.run_query(util.load_query('create_forecast_view.sql', placeholders), workgroup_name) 23 | 24 | return None 25 | -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/create_alter_athena_view/create_aggregate_view.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE VIEW "${database_name}"."carbon_emissions_aggregate_view" AS 2 | SELECT 3 | cet.accountid 4 | , cet.query.querydate 5 | , entries.startdate 6 | , SUM(entries.mbmcarbon) mbmcarbon 7 | , inefficiency.gridmixinefficiency gridmixinefficiency 8 | , inefficiency.servermedianinefficiency servermedianinefficiency 9 | FROM 10 | (("${database_name}"."carbon_emissions_table" cet 11 | CROSS JOIN UNNEST(cet.emissions.carbonemissionentries) t (entries)) 12 | CROSS JOIN UNNEST(cet.emissions.carbonemissionsinefficiency) t (inefficiency)) 13 | WHERE (entries.startdate = inefficiency.startdate) 14 | GROUP BY cet.accountid, cet.query.querydate, entries.startdate, inefficiency.gridmixinefficiency, inefficiency.servermedianinefficiency 15 | -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/create_alter_athena_view/create_forecast_view.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE VIEW "${database_name}"."carbon_emissions_forecast_view" AS 2 | SELECT 3 | cet.accountid 4 | , forecast.carboninefficiency 5 | , forecast.mbmCarbon 6 | , forecast.startDate as years 7 | FROM 8 | ("${database_name}"."carbon_emissions_table" cet 9 | CROSS JOIN UNNEST(cet.emissions.carbonEmissionsForecast) AS t(forecast)) 10 | -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/create_alter_athena_view/create_table.ddl: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS `${database_name}`.`carbon_emissions_table` ( 2 | `accountid` string, 3 | `query` struct, 6 | `emissions` struct>, 7 | carbonEmissionsForecast:array>, 8 | carbonEmissionsInefficiency:array>> ) 10 | ROW FORMAT SERDE 11 | 'org.openx.data.jsonserde.JsonSerDe' 12 | STORED AS INPUTFORMAT 13 | 'org.apache.hadoop.mapred.TextInputFormat' 14 | OUTPUTFORMAT 15 | 'org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat' 16 | LOCATION 17 | '${emissions_location}' -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/create_alter_athena_view/create_view.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE VIEW "${database_name}"."carbon_emissions_view" AS 2 | SELECT 3 | cet.accountid 4 | , cet.query.querydate 5 | , entries.startdate 6 | , entries.mbmcarbon 7 | , entries.paceproductcode 8 | , entries.regioncode 9 | FROM 10 | ("${database_name}"."carbon_emissions_table" cet 11 | CROSS JOIN UNNEST(cet.emissions.carbonemissionentries) t (entries)) 12 | -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/create_alter_athena_view/util.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | import time 4 | from boto3 import client 5 | import logging 6 | 7 | log = logging.getLogger(__name__) 8 | athena = client('athena') 9 | 10 | class AthenaQueryExecutionException(Exception): 11 | pass 12 | 13 | def load_query(filename, placeholders={}): 14 | 15 | with open(filename, 'r', encoding='utf-8') as file: 16 | statement = file.read() 17 | for name, replacement in placeholders.items(): 18 | statement = statement.replace("${%s}" % name, replacement) 19 | return statement 20 | 21 | def wait_for_query_execution(query_execution_id: str): 22 | while True: 23 | response = athena.get_query_execution( 24 | QueryExecutionId=query_execution_id 25 | ) 26 | state = response['QueryExecution']['Status']['State'] 27 | if state == 'SUCCEEDED': 28 | return 29 | elif state == 'FAILED' or state == 'CANCELLED': 30 | raise AthenaQueryExecutionException(f"Query {query_execution_id} failed: {response['QueryExecution']['Status']['StateChangeReason']}") 31 | time.sleep(2) 32 | 33 | def run_query(query: str, workgroup_name: str): 34 | 35 | log.debug(f"Running query: \"{query}\" in workgroup: \"{workgroup_name}\"") 36 | 37 | params = { 38 | 'QueryString': query, 39 | 'WorkGroup': workgroup_name 40 | } 41 | 42 | response = athena.start_query_execution(**params) 43 | return wait_for_query_execution(response['QueryExecutionId']) 44 | -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/extract_carbon_emissions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/experimental-programmatic-access-ccft/d1c273542acee52adedfaf85a44d48d865929a95/MultiAccountApplication/lambda_functions/extract_carbon_emissions/__init__.py -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/extract_carbon_emissions/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | # software and associated documentation files (the "Software"), to deal in the Software 5 | # without restriction, including without limitation the rights to use, copy, modify, 6 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | # permit persons to whom the Software is furnished to do so. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 10 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 11 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 12 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 13 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 14 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | # This resembles the access to AWS Customer Carbon Footprint Tool data 17 | # from the AWS Billing Console. Hence it is not using an official AWS interface and 18 | # might change at any time without notice and just stop working. 19 | 20 | import json 21 | import logging 22 | import os 23 | from urllib.parse import urlencode 24 | 25 | import boto3 26 | import ccft_access 27 | 28 | s3 = boto3.resource("s3") 29 | sts_client = boto3.client("sts") 30 | s3_bucket = os.environ["bucketName"] 31 | role_name = os.environ["ccftRole"] 32 | s3_obj_name = os.environ["fileName"] 33 | 34 | log = logging.getLogger(__name__) 35 | 36 | 37 | def lambda_handler(event, context): 38 | """Lambda handler to retrieve and write ccft data to S3 for the timeframe from start_date to 39 | end_date. 40 | 41 | Args: 42 | event: the AWS Lambda event. 43 | 44 | Must contain: 45 | 46 | - AWS account id as event/account 47 | - query start date as event/timeframe/start_date 48 | - query end date as event/timeframe/end_date 49 | 50 | May contain: 51 | 52 | - skip_write: if set to True, the function will not write the data to S3 53 | 54 | context: the AWS Lambda context 55 | 56 | Raises: 57 | ValueError: if the event does not contain start_date and end_date 58 | 59 | Returns: 60 | isDataAvailable: if data is available for the given timeframe 61 | """ 62 | # get start/ end/ account from the event 63 | # validate argument event[timeframe]['start'] and event[timeframe]['end'] 64 | if ( 65 | "timeframe" not in event 66 | or "end_date" not in event["timeframe"] 67 | or "start_date" not in event["timeframe"] 68 | ): 69 | raise ValueError( 70 | "event must have dates (YYYY-MM-DD) in event/timeframe/start_date and event/timeframe/end_date" 71 | ) 72 | if "account" not in event: 73 | raise ValueError("event must have account-id in event/account") 74 | 75 | skip_write = ("skip_write" in event) and event["skip_write"] 76 | 77 | account = event["account"] 78 | start_date = event["timeframe"]["start_date"] 79 | end_date = event["timeframe"]["end_date"] 80 | 81 | request_id = context.aws_request_id 82 | 83 | # Create a new session and get credentials from target role 84 | target_account_role_arn = f"arn:aws:iam::{account}:role/{role_name}" 85 | target_account_role = sts_client.assume_role( 86 | RoleArn=target_account_role_arn, RoleSessionName="sustainability_reader_session" 87 | ) 88 | 89 | new_access_key_id = target_account_role["Credentials"]["AccessKeyId"] 90 | new_secret_access_key = target_account_role["Credentials"]["SecretAccessKey"] 91 | new_session_token = target_account_role["Credentials"]["SessionToken"] 92 | 93 | new_session = boto3.Session( 94 | aws_access_key_id=new_access_key_id, 95 | aws_secret_access_key=new_secret_access_key, 96 | aws_session_token=new_session_token, 97 | ) 98 | 99 | credentials = new_session.get_credentials() 100 | 101 | try: 102 | log.debug(f"Extracting emissions data from {start_date} to {end_date}") 103 | emissions_data = ccft_access.extract_emissions_data( 104 | start_date, end_date, credentials 105 | ) 106 | 107 | # check if new emissions data is available 108 | carbonEmissionEntries = emissions_data["emissions"]["carbonEmissionEntries"] 109 | if not carbonEmissionEntries: 110 | isDataAvailable = False 111 | message = "No new data is available" 112 | else: 113 | isDataAvailable = True 114 | # save json to file in s3 bucket 115 | if skip_write: 116 | message = f"Skipped saving data for account {account}" 117 | else: 118 | s3_obj_name = ( 119 | f"{account}/{request_id}_{start_date}carbon_emissions.json" 120 | ) 121 | s3.Object(s3_bucket, s3_obj_name).put(Body=json.dumps(emissions_data)) 122 | message = f"Successfully saved data to S3 bucket for account {account}" 123 | return {"message": message, "isDataAvailable": isDataAvailable} 124 | 125 | except Exception as e: 126 | # If account is less than three months old, no carbon emissions data will be available 127 | message = str(e) 128 | 129 | if "404 Client Error: Not Found for url" in str(e): 130 | message = f"""No carbon footprint report is available for account {account} at this time. 131 | If no report is available, your account might be too new to show data. 132 | There is a delay of three months between the end of a month and when emissions data is available.""" 133 | 134 | return {"message": message, "isDataAvailable": False} 135 | -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/extract_carbon_emissions/ccft_access.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | # software and associated documentation files (the "Software"), to deal in the Software 5 | # without restriction, including without limitation the rights to use, copy, modify, 6 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | # permit persons to whom the Software is furnished to do so. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 10 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 11 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 12 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 13 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 14 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | # This resembles the access to AWS Customer Carbon Footprint Tool data 17 | # from the AWS Billing Console. Hence it is not using an official AWS interface and 18 | # might change at any time without notice and just stop working. 19 | 20 | import argparse 21 | import json 22 | import sys 23 | from datetime import datetime, timezone 24 | 25 | import boto3 26 | import requests 27 | from botocore.auth import SigV4Auth 28 | from botocore.awsrequest import AWSRequest 29 | from dateutil.relativedelta import relativedelta 30 | 31 | 32 | def extract_emissions_data(startDate, endDate, credentials): 33 | billing_region = "us-east-1" 34 | host = f"{billing_region}.prod.sustainability.billingconsole.aws.dev" 35 | url = f"https://{host}/get-carbon-footprint-summary?startDate={startDate}&endDate={endDate}" 36 | method = "GET" 37 | 38 | if credentials.token is None: 39 | # this is most likely an IAM or root user 40 | exit("You seem to run this with an IAM user. Assume an account's role instead.") 41 | 42 | # get the account ID to include it in the response 43 | sts_client = boto3.client( 44 | "sts", 45 | aws_access_key_id=credentials.access_key, 46 | aws_secret_access_key=credentials.secret_key, 47 | aws_session_token=credentials.token, 48 | ) 49 | 50 | accountID = sts_client.get_caller_identity()["Account"] 51 | 52 | request = AWSRequest(method, url, headers={"Host": host}) 53 | 54 | SigV4Auth(credentials, "sustainability", billing_region).add_auth(request) 55 | 56 | try: 57 | response = requests.request( 58 | method, url, headers=dict(request.headers), data={}, timeout=5 59 | ) 60 | response.raise_for_status() 61 | except Exception as e: 62 | if "404 Client Error: Not Found for url" in str(e): 63 | raise Exception( 64 | f"""No carbon footprint report is available for account {accountID} at this time. 65 | If no report is available, your account might be too new to show data. 66 | There is a delay of three months between the end of a month and when emissions data is available.""" 67 | ) 68 | else: 69 | raise Exception("An error occured: " + str(e)) 70 | 71 | emissions = response.json() 72 | 73 | emissions_data = { 74 | "accountId": accountID, 75 | "query": { 76 | "queryDate": datetime.now(timezone.utc).strftime("%Y-%m-%d"), 77 | "startDate": startDate, 78 | "endDate": endDate, 79 | }, 80 | "emissions": emissions, 81 | } 82 | 83 | print(json.dumps(emissions_data)) 84 | return emissions_data 85 | 86 | 87 | if __name__ == "__main__": 88 | parser = argparse.ArgumentParser( 89 | description="Script to extract carbon emissions from the AWS Customer Carbon Footprint Tool." 90 | ) 91 | 92 | # calculate three months past (when new carbon emissions data is available) 93 | three_months = datetime.now(timezone.utc) - relativedelta(months=3) 94 | # get the date with 1st day of the month 95 | year = three_months.year 96 | month = three_months.month 97 | first_date = datetime(year, month, 1) 98 | 99 | # The default end date is the first date of the month three months ago, and the default start date is 36 months before 100 | default_end_date = first_date.strftime("%Y-%m-%d") 101 | default_start_date = (first_date - relativedelta(months=36)).strftime("%Y-%m-%d") 102 | 103 | parser = argparse.ArgumentParser( 104 | description="""Experimental retrieval of AWS Customer Carbon Footprint Tool console data. 105 | The data is queried for a closed interval from START_DATE to END_DATE (YYYY-MM-DD). 106 | The queried timeframe must be less than 36 months and not before 2020-01-01.""" 107 | ) 108 | parser.add_argument( 109 | "--start-date", 110 | "-s", 111 | type=lambda s: datetime.strptime(s, "%Y-%m-%d"), 112 | default=datetime.strptime(default_start_date, "%Y-%m-%d"), 113 | help=f"first month of the closed interval, default 36 months before end date: {default_start_date}", 114 | ) 115 | parser.add_argument( 116 | "--end-date", 117 | "-e", 118 | type=lambda s: datetime.strptime(s, "%Y-%m-%d"), 119 | default=datetime.strptime(default_end_date, "%Y-%m-%d"), 120 | help=f"last month of the closed interval, default 3 months before current date: {default_end_date}", 121 | ) 122 | 123 | args = parser.parse_args() 124 | start_date = args.start_date.strftime("%Y-%m-%d") 125 | end_date = args.end_date.strftime("%Y-%m-%d") 126 | 127 | session = boto3.Session() 128 | credentials = session.get_credentials() 129 | 130 | if credentials is None: 131 | exit( 132 | 'You need to configure an AWS profile with an IAM role to run this script (see FAQ "How do I use the script?").' 133 | ) 134 | 135 | try: 136 | extract_emissions_data(start_date, end_date, credentials) 137 | 138 | except Exception as e: 139 | sys.stderr.write(str(e)) 140 | sys.exit(1) 141 | -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/extract_carbon_emissions/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | python-dateutil 2 | boto3 -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/extract_carbon_emissions/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | urllib3<2 -------------------------------------------------------------------------------- /MultiAccountApplication/lambda_functions/get_account_ids/app.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from datetime import datetime, date 3 | import logging 4 | from dateutil.relativedelta import relativedelta 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | def create_timeframes(today:str): 9 | """Calculate the timeframes for the backfill and the checking/ loading of new data. 10 | 11 | Args: 12 | today (str): today in the format of YYYY-MM-DD 13 | """ 14 | today_date = datetime.strptime(today, '%Y-%m-%d').date() 15 | 16 | # 1. new data: calculate three months past (when new carbon emissions data 17 | # is available) 18 | three_months = today_date - relativedelta(months=3) 19 | # get the date with 1st day of the month 20 | # the start data will be used for both as start and end of a query 21 | start_new_data = three_months.replace(day=1).strftime("%Y-%m-%d") 22 | 23 | # 2. backfill: since the maximum time to extract carbon emissions data is 36 24 | # months, we can use the timeframe from 40-4 months back 25 | four_months = today_date - relativedelta(months=4) 26 | forty_months = today_date - relativedelta(months=40) 27 | 28 | start_backfill = forty_months.replace(day=1).strftime("%Y-%m-%d") 29 | end_backfill = four_months.replace(day=1).strftime("%Y-%m-%d") 30 | 31 | return { 'timeframes': { 32 | 'backfill': { 33 | 'start_date': start_backfill, 34 | 'end_date': end_backfill 35 | }, 36 | 'new_data': { 37 | 'start_date': start_new_data, 38 | 'end_date': start_new_data 39 | } 40 | }} 41 | 42 | def lambda_handler(event, _): 43 | 44 | today = date.today().strftime("%Y-%m-%d") 45 | 46 | if ('override_today' in event): 47 | today = event['override_today'] 48 | 49 | log.debug(f"Pulling data relative to {today}") 50 | 51 | timeframes = create_timeframes(today) 52 | 53 | accounts = [] 54 | 55 | if ('override_accounts' in event): 56 | accounts = event['override_accounts'] 57 | else: 58 | # Create the AWS Organizations client 59 | org_client = boto3.client("organizations") 60 | 61 | # Get the paginator for the list_accounts operation 62 | paginator = org_client.get_paginator("list_accounts") 63 | 64 | # Retrieve the ID of the management account 65 | management_account_id = org_client.describe_organization()['Organization']['MasterAccountId'] 66 | 67 | # Retrieve all active account IDs in the organization (filtering out accounts in status SUSPENDED or PENDING_CLOSURE) 68 | accounts = [ 69 | account["Id"] 70 | for page in paginator.paginate() 71 | for account in page["Accounts"] 72 | if account["Status"] == "ACTIVE" # Filter for active accounts 73 | ] 74 | 75 | # move the management account to the first index 76 | accounts.remove(management_account_id) 77 | accounts = [management_account_id] + accounts 78 | 79 | return { 80 | 'account_ids': accounts 81 | } | timeframes 82 | -------------------------------------------------------------------------------- /MultiAccountApplication/statemachine/extract_carbon_emissions.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "A state machine that extracts carbon emissions data", 3 | "StartAt": "Parallel", 4 | "States": { 5 | "Parallel": { 6 | "Type": "Parallel", 7 | "End": true, 8 | "Branches": [ 9 | { 10 | "StartAt": "CreateAthenaTableView", 11 | "States": { 12 | "CreateAthenaTableView": { 13 | "Type": "Task", 14 | "Resource": "${CreateAlterAthenaViewFunctionArn}", 15 | "ResultPath": "$.athena", 16 | "End": true 17 | } 18 | } 19 | }, 20 | { 21 | "StartAt": "GetAccountIDs", 22 | "States": { 23 | "GetAccountIDs": { 24 | "Type": "Task", 25 | "Resource": "${GetAccountIDsFunctionArn}", 26 | "Next": "CheckFirstInvocation" 27 | }, 28 | "CheckFirstInvocation": { 29 | "Type": "Task", 30 | "Resource": "${CheckFirstInvocationFunctionArn}", 31 | "ResultPath": "$.SecondStateResult", 32 | "Next": "Choice: First Invocation" 33 | }, 34 | "Choice: First Invocation": { 35 | "Type": "Choice", 36 | "InputPath": "$", 37 | "Choices": [ 38 | { 39 | "Variable": "$.SecondStateResult.isEmpty", 40 | "BooleanEquals": true, 41 | "Next": "ProcessAccounts-Backfill" 42 | }, 43 | { 44 | "Variable": "$.SecondStateResult.isEmpty", 45 | "BooleanEquals": false, 46 | "Next": "ExtractCarbonEmissions-Canary" 47 | } 48 | ] 49 | }, 50 | "ProcessAccounts-Backfill": { 51 | "Type": "Map", 52 | "InputPath": "$", 53 | "ItemsPath": "$.account_ids", 54 | "ItemSelector": { 55 | "account.$": "$$.Map.Item.Value", 56 | "timeframe.$": "$.timeframes.backfill" 57 | }, 58 | "ResultPath": null, 59 | "MaxConcurrency": 50, 60 | "ItemProcessor": { 61 | "ProcessorConfig": { 62 | "Mode": "DISTRIBUTED", 63 | "ExecutionType": "STANDARD" 64 | }, 65 | "StartAt": "ExtractCarbonEmissions-Backfill", 66 | "States": { 67 | "ExtractCarbonEmissions-Backfill": { 68 | "Type": "Task", 69 | "Resource": "${ExtractCarbonEmissionsFunctionArn}", 70 | "End": true, 71 | "Catch": [ 72 | { 73 | "ErrorEquals": [ 74 | "States.ALL" 75 | ], 76 | "Next": "HandleError-Process Accounts1" 77 | } 78 | ] 79 | }, 80 | "HandleError-Process Accounts1": { 81 | "Type": "Pass", 82 | "Result": "The ProcessAccounts Lambda function failed.", 83 | "End": true 84 | } 85 | } 86 | }, 87 | "Next": "ExtractCarbonEmissions-Canary" 88 | }, 89 | "ExtractCarbonEmissions-Canary": { 90 | "Type": "Task", 91 | "InputPath": "$", 92 | "Parameters": { 93 | "account.$": "$.account_ids[0]", 94 | "timeframe.$": "$.timeframes.new_data", 95 | "skip_write": true 96 | }, 97 | "Resource": "${ExtractCarbonEmissionsFunctionArn}", 98 | "Catch": [ 99 | { 100 | "ErrorEquals": [ 101 | "States.ALL" 102 | ], 103 | "Next": "HandleError" 104 | } 105 | ], 106 | "Next": "Choice: New Data Available", 107 | "ResultPath": "$.FourthStateResult" 108 | }, 109 | "Choice: New Data Available": { 110 | "Type": "Choice", 111 | "Choices": [ 112 | { 113 | "Variable": "$.FourthStateResult.isDataAvailable", 114 | "BooleanEquals": true, 115 | "Next": "ProcessAccounts" 116 | }, 117 | { 118 | "Variable": "$.FourthStateResult.isDataAvailable", 119 | "BooleanEquals": false, 120 | "Next": "RetryLambdaDaily" 121 | } 122 | ], 123 | "OutputPath": "$" 124 | }, 125 | "RetryLambdaDaily": { 126 | "Type": "Wait", 127 | "Seconds": 86400, 128 | "Next": "ExtractCarbonEmissions-Canary" 129 | }, 130 | "ProcessAccounts": { 131 | "Type": "Map", 132 | "InputPath": "$", 133 | "ItemsPath": "$.account_ids", 134 | "ResultPath": null, 135 | "MaxConcurrency": 50, 136 | "ItemSelector": { 137 | "account.$": "$$.Map.Item.Value", 138 | "timeframe.$": "$.timeframes.new_data" 139 | }, 140 | "ItemProcessor": { 141 | "ProcessorConfig": { 142 | "Mode": "DISTRIBUTED", 143 | "ExecutionType": "STANDARD" 144 | }, 145 | "StartAt": "ExtractCarbonEmissions-All", 146 | "States": { 147 | "ExtractCarbonEmissions-All": { 148 | "Type": "Task", 149 | "Resource": "arn:aws:states:::lambda:invoke", 150 | "OutputPath": "$.Payload.message", 151 | "Parameters": { 152 | "Payload.$": "$", 153 | "FunctionName": "${ExtractCarbonEmissionsFunctionArn}" 154 | }, 155 | "End": true, 156 | "Catch": [ 157 | { 158 | "ErrorEquals": [ 159 | "States.ALL" 160 | ], 161 | "Next": "HandleError-Process Accounts" 162 | } 163 | ] 164 | }, 165 | "HandleError-Process Accounts": { 166 | "Type": "Pass", 167 | "ResultPath": "$.message", 168 | "Result": "Check that you have the necessary permissions and the role exists.", 169 | "End": true 170 | } 171 | } 172 | }, 173 | "End": true 174 | }, 175 | "HandleError": { 176 | "Type": "Pass", 177 | "End": true 178 | } 179 | } 180 | } 181 | ] 182 | } 183 | } 184 | } -------------------------------------------------------------------------------- /MultiAccountApplication/template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: AWS::Serverless-2016-10-31 3 | Metadata: 4 | AWS::ServerlessRepo::Application: 5 | Name: experimental-programmatic-access-ccft 6 | Description: This application automates the monthly extraction of new AWS Customer Carbon Footprint Tool data within a multi account structure 7 | Author: experimental-programmatic-access-ccft-team 8 | LicenseUrl: ../LICENSE 9 | ReadmeUrl: ../README.md 10 | SemanticVersion: 1.2.0 11 | HomePageUrl: https://github.com/aws-samples/experimental-programmatic-access-ccft 12 | SourceCodeUrl: https://github.com/aws-samples/experimental-programmatic-access-ccft 13 | Description: > 14 | ccft-sam-script 15 | 16 | SAM Template to automate the monthly extraction of new CCFT data within a multi account structure (uksb-880c16twdr). 17 | 18 | # More info about Globals: 19 | Globals: 20 | Function: 21 | Timeout: 3 22 | MemorySize: 128 23 | 24 | Parameters: 25 | CarbonEmissionsDataBucketName: 26 | Description: "Name suffix (w/o prefixes for acct ID, region) for carbon emissions data bucket (will be created by this stack)" 27 | Type: "String" 28 | Default: "ccft-data" 29 | CarbonEmissionsDataFileName: 30 | Description: "Name prefix for the .json file as output of the script" 31 | Type: "String" 32 | Default: "ccft-access.json" 33 | CCFTRoleName: 34 | Description: "Name of the IAM role that was deployed into all member accounts and gives read access to the AWS CCFT data" 35 | Type: "String" 36 | Default: "ccft-read-role" 37 | GlueDatabaseName: 38 | Description: "Name of the Glue database used for Athena (will be created by this stack)" 39 | Type: "String" 40 | Default: "carbon_emissions" 41 | 42 | Resources: 43 | 44 | # ------------------------------------------------------------------------------------------------------------------- 45 | # Create an S3 bucket to be used as data storage for AWS Customer Carbon Footprint Tool data. 46 | # ------------------------------------------------------------------------------------------------------------------- 47 | 48 | CarbonEmissionsDataBucket: 49 | Type: AWS::S3::Bucket 50 | Metadata: 51 | guard: 52 | SuppressedRules: 53 | # We neither log nor version the bucket for the code sample to reduce storage. 54 | # Do versioning and lifecycling of deleted versions as needed. 55 | - S3_BUCKET_LOGGING_ENABLED 56 | - S3_BUCKET_VERSIONING_ENABLED 57 | Properties: 58 | BucketName: !Sub "${AWS::AccountId}-${AWS::Region}-${CarbonEmissionsDataBucketName}" 59 | PublicAccessBlockConfiguration: 60 | BlockPublicAcls: true 61 | BlockPublicPolicy: true 62 | IgnorePublicAcls: true 63 | RestrictPublicBuckets: true 64 | BucketEncryption: 65 | ServerSideEncryptionConfiguration: 66 | - ServerSideEncryptionByDefault: 67 | SSEAlgorithm: AES256 68 | 69 | CarbonEmissionsDataBucketPolicy: 70 | Type: AWS::S3::BucketPolicy 71 | Properties: 72 | Bucket: !Ref CarbonEmissionsDataBucket 73 | PolicyDocument: 74 | Statement: 75 | - Effect: Deny 76 | Principal: "*" 77 | Action: "s3:*" 78 | Resource: 79 | - !Sub "arn:aws:s3:::${CarbonEmissionsDataBucket}" 80 | - !Sub "arn:aws:s3:::${CarbonEmissionsDataBucket}/*" 81 | Condition: 82 | Bool: 83 | aws:SecureTransport: false 84 | 85 | # ------------------------------------------------------------------------------------------------------------------- 86 | # Create an S3 bucket to be used as Athena output location, with a retention time of files of 1 day. 87 | # ------------------------------------------------------------------------------------------------------------------- 88 | 89 | AthenaResultBucket: 90 | Type: AWS::S3::Bucket 91 | Metadata: 92 | guard: 93 | SuppressedRules: 94 | # We neither log nor version the bucket for the code sample to reduce storage. 95 | # Do versioning and lifecycling of deleted versions as needed. 96 | - S3_BUCKET_LOGGING_ENABLED 97 | - S3_BUCKET_VERSIONING_ENABLED 98 | Properties: 99 | BucketName: !Sub "${AWS::AccountId}-${AWS::Region}-athenaresults" 100 | LifecycleConfiguration: 101 | Rules: 102 | - Id: DeleteAfterOneDay 103 | Status: Enabled 104 | ExpirationInDays: 1 105 | PublicAccessBlockConfiguration: 106 | BlockPublicAcls: true 107 | BlockPublicPolicy: true 108 | IgnorePublicAcls: true 109 | RestrictPublicBuckets: true 110 | BucketEncryption: 111 | ServerSideEncryptionConfiguration: 112 | - ServerSideEncryptionByDefault: 113 | SSEAlgorithm: AES256 114 | 115 | AthenaResultBucketPolicy: 116 | Type: AWS::S3::BucketPolicy 117 | Properties: 118 | Bucket: !Ref AthenaResultBucket 119 | PolicyDocument: 120 | Statement: 121 | - Effect: Deny 122 | Principal: "*" 123 | Action: "s3:*" 124 | Resource: 125 | - !Sub "arn:aws:s3:::${AthenaResultBucket}" 126 | - !Sub "arn:aws:s3:::${AthenaResultBucket}/*" 127 | Condition: 128 | Bool: 129 | aws:SecureTransport: false 130 | 131 | 132 | # ------------------------------------------------------------------------------------------------------------------- 133 | # Create an AWS Step Function State Machine and a EventBridge rule to trigger the Step Functions workflow. 134 | # ------------------------------------------------------------------------------------------------------------------- 135 | 136 | ExtractCarbonEmissionsStateMachine: 137 | Type: AWS::Serverless::StateMachine 138 | Properties: 139 | DefinitionUri: statemachine/extract_carbon_emissions.asl.json 140 | DefinitionSubstitutions: 141 | ExtractCarbonEmissionsFunctionArn: !GetAtt ExtractCarbonEmissionsFunction.Arn 142 | GetAccountIDsFunctionArn: !GetAtt GetAccountIDsFunction.Arn 143 | CreateAlterAthenaViewFunctionArn: !GetAtt CreateAlterAthenaViewFunction.Arn 144 | CheckFirstInvocationFunctionArn: !GetAtt CheckFirstInvocationFunction.Arn 145 | CarbonEmissionsDataBucketArn: !GetAtt CarbonEmissionsDataBucket.Arn 146 | CarbonEmissionsDataBucketName: !Ref CarbonEmissionsDataBucket 147 | Events: 148 | ComplexScheduleEvent: 149 | Type: ScheduleV2 150 | Properties: 151 | ScheduleExpression: "cron(0 0 15 * ? *)" 152 | FlexibleTimeWindow: 153 | Mode: FLEXIBLE 154 | MaximumWindowInMinutes: 60 155 | Policies: 156 | - LambdaInvokePolicy: 157 | FunctionName: !Ref ExtractCarbonEmissionsFunction 158 | - LambdaInvokePolicy: 159 | FunctionName: !Ref GetAccountIDsFunction 160 | - LambdaInvokePolicy: 161 | FunctionName: !Ref CreateAlterAthenaViewFunction 162 | - LambdaInvokePolicy: 163 | FunctionName: !Ref CheckFirstInvocationFunction 164 | - Statement: 165 | - Sid: AllowStartExecution 166 | Effect: Allow 167 | Action: 168 | - states:StartExecution 169 | Resource: '*' 170 | 171 | # ------------------------------------------------------------------------------------------------------------------- 172 | # Create a Lambda function to get account IDs 173 | # ------------------------------------------------------------------------------------------------------------------- 174 | 175 | GetAccountIDsFunction: 176 | Type: AWS::Serverless::Function 177 | Properties: 178 | FunctionName: "get-account-ids" 179 | CodeUri: lambda_functions/get_account_ids/ 180 | Handler: app.lambda_handler 181 | Policies: 182 | - Statement: 183 | - Sid: ListAccounts 184 | Effect: Allow 185 | Action: 186 | - organizations:ListAccounts 187 | Resource: "*" 188 | - Sid: DescribeOrganization 189 | Effect: Allow 190 | Action: 191 | - organizations:DescribeOrganization 192 | Resource: "*" 193 | Runtime: python3.11 194 | Architectures: 195 | - arm64 196 | Timeout: 20 197 | 198 | GetAccountIDsFunctionLogGroup: 199 | Type: AWS::Logs::LogGroup 200 | Metadata: 201 | guard: 202 | SuppressedRules: 203 | # Log group data is always encrypted in CloudWatch Logs. 204 | # By default, CloudWatch Logs uses server-side encryption for the log data at rest. 205 | - CLOUDWATCH_LOG_GROUP_ENCRYPTED 206 | Properties: 207 | LogGroupName: !Sub "/aws/lambda/${GetAccountIDsFunction}" 208 | RetentionInDays: 1 209 | 210 | # ------------------------------------------------------------------------------------------------------------------- 211 | # Create a Lambda function to check for data for first invocation 212 | # ------------------------------------------------------------------------------------------------------------------- 213 | 214 | CheckFirstInvocationFunction: 215 | Type: AWS::Serverless::Function 216 | Properties: 217 | FunctionName: "check-first-invocation" 218 | CodeUri: lambda_functions/check_first_invocation/ 219 | Handler: app.lambda_handler 220 | Policies: 221 | - S3ReadPolicy: 222 | BucketName: !Ref CarbonEmissionsDataBucket 223 | - Statement: 224 | - Sid: ListAccounts 225 | Effect: Allow 226 | Action: 227 | - organizations:ListAccounts 228 | Resource: "*" 229 | - Sid: DescribeOrganization 230 | Effect: Allow 231 | Action: 232 | - organizations:DescribeOrganization 233 | Resource: "*" 234 | Runtime: python3.11 235 | Architectures: 236 | - arm64 237 | Environment: 238 | Variables: 239 | bucketName: !Ref CarbonEmissionsDataBucket 240 | Timeout: 20 241 | 242 | CheckFirstInvocationFunctionLogGroup: 243 | Type: AWS::Logs::LogGroup 244 | Metadata: 245 | guard: 246 | SuppressedRules: 247 | # Log group data is always encrypted in CloudWatch Logs. 248 | # By default, CloudWatch Logs uses server-side encryption for the log data at rest. 249 | - CLOUDWATCH_LOG_GROUP_ENCRYPTED 250 | Properties: 251 | LogGroupName: !Sub "/aws/lambda/${CheckFirstInvocationFunction}" 252 | RetentionInDays: 1 253 | 254 | # ------------------------------------------------------------------------------------------------------------------- 255 | # Create a Lambda function, role + log group to extract AWS Customer Carbon Footprint Tool data. 256 | # ------------------------------------------------------------------------------------------------------------------- 257 | 258 | ExtractCarbonEmissionsLambdaRole: 259 | Type: AWS::IAM::Role 260 | Metadata: 261 | guard: 262 | SuppressedRules: 263 | - IAM_NO_INLINE_POLICY_CHECK 264 | Properties: 265 | RoleName: extract-emissions-lambda-role 266 | AssumeRolePolicyDocument: 267 | Version: "2012-10-17" 268 | Statement: 269 | - Effect: Allow 270 | Principal: 271 | Service: lambda.amazonaws.com 272 | Action: sts:AssumeRole 273 | Policies: 274 | - PolicyName: CarbonDataExtractionPolicy 275 | PolicyDocument: 276 | Version: "2012-10-17" 277 | Statement: 278 | - Sid: GetCarbonFootprintSummary 279 | Effect: Allow 280 | Action: 281 | - sustainability:GetCarbonFootprintSummary 282 | Resource: '*' 283 | - Sid: S3WritePolicy 284 | Effect: Allow 285 | Action: 286 | - s3:PutObject 287 | Resource: !Sub "${CarbonEmissionsDataBucket.Arn}/*" 288 | - Sid: STSAssumeRole 289 | Effect: Allow 290 | Action: 291 | - sts:AssumeRole 292 | Resource: !Sub "arn:${AWS::Partition}:iam::*:role/${CCFTRoleName}" 293 | 294 | ExtractCarbonEmissionsFunction: 295 | Type: AWS::Serverless::Function 296 | Properties: 297 | FunctionName: "extract-carbon-emissions-data" 298 | CodeUri: lambda_functions/extract_carbon_emissions/ 299 | Handler: app.lambda_handler 300 | Role: !GetAtt ExtractCarbonEmissionsLambdaRole.Arn 301 | Runtime: python3.11 302 | Architectures: 303 | - arm64 304 | Environment: 305 | Variables: 306 | bucketName: !Ref CarbonEmissionsDataBucket 307 | ccftRole: !Ref CCFTRoleName 308 | fileName: !Ref CarbonEmissionsDataFileName 309 | Timeout: 20 310 | 311 | ExtractCarbonEmissionsFunctionLogGroup: 312 | Type: AWS::Logs::LogGroup 313 | Metadata: 314 | guard: 315 | SuppressedRules: 316 | # Log group data is always encrypted in CloudWatch Logs. 317 | # By default, CloudWatch Logs uses server-side encryption for the log data at rest. 318 | - CLOUDWATCH_LOG_GROUP_ENCRYPTED 319 | Properties: 320 | LogGroupName: !Sub "/aws/lambda/${ExtractCarbonEmissionsFunction}" 321 | RetentionInDays: 1 322 | 323 | AthenaWorkgroup: 324 | Type: AWS::Athena::WorkGroup 325 | Properties: 326 | Name: !Sub "${GlueDatabaseName}-workgroup" 327 | Description: !Sub "Workgroup for queries to the ${GlueDatabaseName} database by the experimental programmatic access" 328 | RecursiveDeleteOption: true 329 | State: "ENABLED" 330 | WorkGroupConfiguration: 331 | EnforceWorkGroupConfiguration: true 332 | ResultConfiguration: 333 | OutputLocation: !Sub "s3://${AthenaResultBucket}" 334 | 335 | # ------------------------------------------------------------------------------------------------------------------- 336 | # Create a Lambda function to create Athena table and view. 337 | # ------------------------------------------------------------------------------------------------------------------- 338 | 339 | CreateAlterAthenaViewFunction: 340 | Type: AWS::Serverless::Function 341 | Properties: 342 | FunctionName: "create-alter-athena-view" 343 | CodeUri: lambda_functions/create_alter_athena_view/ 344 | Handler: app.lambda_handler 345 | Policies: 346 | - Statement: 347 | - Sid: Athena 348 | Effect: Allow 349 | Action: 350 | - athena:StartQueryExecution 351 | - athena:GetQueryResults 352 | - athena:StopQueryExecution 353 | - athena:GetQueryExecution 354 | - glue:CreateDatabase 355 | - glue:GetDatabase 356 | - glue:CreateTable 357 | - glue:UpdateTable 358 | - glue:GetTable 359 | Resource: 360 | - !Sub "arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:workgroup/${AthenaWorkgroup}" 361 | - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog" 362 | - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${GlueDatabaseName}" 363 | - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${GlueDatabaseName}/*" 364 | - Sid: S3Allow 365 | Effect: Allow 366 | Action: 367 | - s3:* 368 | Resource: 369 | - !Sub "${CarbonEmissionsDataBucket.Arn}/*" 370 | - !Sub "${AthenaResultBucket.Arn}/*" 371 | - !Sub "${AthenaResultBucket.Arn}" 372 | Runtime: python3.11 373 | Architectures: 374 | - arm64 375 | Environment: 376 | Variables: 377 | emissionsBucketName: !Ref CarbonEmissionsDataBucket 378 | workgroupName: !Ref AthenaWorkgroup 379 | glueDatabaseName: !Ref GlueDatabaseName 380 | Timeout: 20 381 | -------------------------------------------------------------------------------- /MultiAccountApplication/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/experimental-programmatic-access-ccft/d1c273542acee52adedfaf85a44d48d865929a95/MultiAccountApplication/tests/__init__.py -------------------------------------------------------------------------------- /MultiAccountApplication/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | -------------------------------------------------------------------------------- /MultiAccountApplication/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/experimental-programmatic-access-ccft/d1c273542acee52adedfaf85a44d48d865929a95/MultiAccountApplication/tests/unit/__init__.py -------------------------------------------------------------------------------- /MultiAccountApplication/tests/unit/test.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE VIEW "${mykey}" AS ... 2 | CREATE OR REPLACE VIEW "${myotherkey}" AS ... -------------------------------------------------------------------------------- /MultiAccountApplication/tests/unit/test_create_alter_athena_view.py: -------------------------------------------------------------------------------- 1 | import os 2 | from lambda_functions.create_alter_athena_view import util 3 | from pathlib import Path 4 | 5 | def test_util(): 6 | 7 | test_statement = util.load_query(Path(__file__).parent.joinpath('test.sql'), 8 | {"mykey": "myplaceholdervalue", 9 | "myotherkey": "myotherplaceholder"}) 10 | 11 | assert test_statement == "CREATE OR REPLACE VIEW \"myplaceholdervalue\" AS ...\nCREATE OR REPLACE VIEW \"myotherplaceholder\" AS ..." 12 | -------------------------------------------------------------------------------- /MultiAccountApplication/tests/unit/test_get_account_ids.py: -------------------------------------------------------------------------------- 1 | from lambda_functions.get_account_ids import app 2 | 3 | 4 | def test_create_timeframes(): 5 | 6 | assert app.create_timeframes("2024-07-04") == { 7 | "timeframes": { 8 | "new_data": {"start_date": "2024-04-01", "end_date": "2024-04-01"}, 9 | "backfill": {"start_date": "2021-03-01", "end_date": "2024-03-01"}, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ⚠️ Deprecation Warning 2 | 3 | On April 24th 2025 [AWS launched Carbon emissions Data Exports](https://aws.amazon.com/about-aws/whats-new/2025/04/customer-carbon-footprint-tool-updated-methodology/), a managed export feature for carbon emissions estimates. The endpoint used by this sample code will be discontinued on July 23rd 2025 and this sample code will stop working. We recommend using Carbon emissions Data Exports instead. Read [Access and visualize carbon emissions data from AWS Data Exports](https://aws.amazon.com/blogs/aws-cloud-financial-management/export-and-visualize-carbon-emissions-data-from-your-aws-accounts/) for details and instructions on how to source the data from Data Exports and visualize it. 4 | 5 | # Experimental programmatic access to the AWS Customer Carbon Footprint Tool data 6 | 7 | You can use the [AWS Customer Carbon Footprint Tool](https://aws.amazon.com/aws-cost-management/aws-customer-carbon-footprint-tool/) (CCFT) to view estimates of the carbon emissions associated with your AWS products and services. You can access the same AWS Customer Carbon Footprint Tool information by resembling the behavior of the console with this **experimental** script. 8 | 9 | The script can be used for programmatic access to the same AWS Customer Carbon Footprint Tool data the browser has access to. It enables customers to do two things: 10 | 1. Programmatic access to feasibly get individual estimates of hundreds or thousands of accounts without logging in to each account manually. 11 | 2. Lowered carbon reporting threshold to kilogram level (three decimal digits) as introduced in the [CSV file download feature](https://aws.amazon.com/blogs/aws-cloud-financial-management/increased-visibility-of-your-carbon-emissions-data-with-aws-customer-carbon-footprint-tool/). 12 | 13 | This repository gives you supporting source code for two use cases: 14 | 1. If you are looking for a way to extract CCFT data for a small number of accounts on an ad-hoc basis, or want to include the script within your application, you can find the [`ccft_access.py`](./MultiAccountApplication/lambda_functions/extract_carbon_emissions/ccft_access.py) script itself in the [`MultiAccountApplication/lambda_functions/extract_carbon_emissions/`](./MultiAccountApplication/lambda_functions/extract_carbon_emissions/) folder. To get started, check out the [General FAQs](#general-faq) and the [single-account specific FAQs](#single-account-script-faq) below. 15 | 16 | 2. If you are looking for a way to automate the monthly extraction of new CCFT data within a multi account structure, this repository contains source code and supporting files for a serverless application that you can deploy with the SAM CLI or via the Serverless Application Repository. With it, you can deploy an application to extract new AWS Customer Carbon Footprint Tool data every month for all accounts of your AWS organization with the experimental script. You can find the supporting source code within the folder [`MultiAccountApplication`](./MultiAccountApplication). To get started, check out the [General FAQs](#general-faq) and the [multi-account specific FAQs](#multi-account-extraction-faq) below. 17 | 18 | Read the AWS Customer Carbon Footprint Tool documentation for more details to [understand your carbon emission estimations](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/ccft-estimation.html). 19 | 20 | ## Security 21 | 22 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 23 | 24 | ## License 25 | 26 | This library is licensed under the MIT-0 License. See the LICENSE file. 27 | 28 | ## FAQ 29 | 30 | ## General FAQ 31 | 32 | ### Q: What does experimental mean? 33 | 34 | This script resembles the access to CCFT data from the AWS Billing Console. Hence it is not using an official AWS interface and might change at any time without notice and just stop working. 35 | 36 | ### Q: How does the data relate to what I see in the AWS Customer Carbon Footprint Tool? 37 | 38 | On a high-level, the output from calling the experimental programmatic access script looks like the following. See the respective numbering in the screenshot of the AWS Customer Carbon Footprint Tool below to understand where you can find the respective information. 39 | ```bash 40 | { 41 | "accountId": "████████████", 42 | "query": { 43 | "queryDate": <-- date when query was executed 44 | "startDate": <-- START_DATE of query 45 | "endDate": <-- END_DATE of query 46 | }, 47 | "emissions": { 48 | "carbonEmissionEntries": [ 49 | { 50 | "mbmCarbon": <-- (1), Your estimated carbon emissions in metric tons of CO2eq, following the market-based method (mbm) of the Greenhouse Gas Protocol 51 | "paceProductCode": <-- (2), Your emissions by service 52 | "regionCode": <-- (3), Your emissions by geography 53 | "startDate": <-- month this data relates to 54 | }, 55 | { 56 | […] 57 | } 58 | ], 59 | "carbonEmissionsForecast": [ 60 | { 61 | […] 62 | "mbmCarbon": <-- Your estimated, forecasted carbon emissions in metric tons of CO2eq, following the market-based method (mbm) of the Greenhouse Gas Protocol 63 | "startDate": <-- year this data relates to 64 | }, 65 | { 66 | […] 67 | } 68 | ], 69 | "carbonEmissionsInefficiency": [ 70 | { 71 | "gridMixInefficiency": <-- (4), Estimated emissions savings: difference between the carbon footprint emissions calculated using the location-based method (LBM) and the market-based method (MBM). 72 | […] 73 | "startDate": <-- month this data relates to 74 | }, 75 | { 76 | […] 77 | } 78 | ] 79 | […] 80 | ``` 81 | ![Console Reference](static/ccft-api-explanation.png) 82 | 83 | If your AWS Customer Carbon Footprint Tool emissions are zero, the script will also return `0.0`. Please note, that you will not see the product split or region split in this case (`paceProductCode` and `regionCode` under `carbonEmissionEntries` will not be returned). 84 | 85 | Read the AWS Customer Carbon Footprint Tool documentation for more details to [understand your carbon emission estimations](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/ccft-estimation.html). 86 | 87 | 88 | ## Single-account script FAQ 89 | 90 | ### Q: How do I use the script? 91 | 92 | 1. Clone the repository and navigate to the folder [`MultiAccountApplication/lambda_functions/extract_carbon_emissions/`](./MultiAccountApplication/lambda_functions/extract_carbon_emissions/). 93 | 2. [Assume](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html) a [role with access](#q-what-aws-iam-role-do-i-need) to the AWS Customer Carbon Footprint Tool. 94 | 3. Execute the script: 95 | 96 | ```bash 97 | python ccft_access.py 98 | ``` 99 | 100 | ```bash 101 | { 102 | "accountId": "████████████", 103 | "query": { 104 | "queryDate": "2023-02-12", "startDate": "2020-01-01", "endDate": "2023-01-01" 105 | }, 106 | "emissions": { 107 | "carbonEmissionEntries": [ 108 | { 109 | "mbmCarbon": "0.048", "paceProductCode": "Other", "regionCode": "EMEA", "startDate": "2020-01-01" 110 | }, 111 | […] 112 | ``` 113 | 114 | ### Q: What AWS IAM role do I need? 115 | 116 | Use a role with the following AWS IAM policy that contains the [AWS Customer Carbon Footprint Tool IAM permission](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/what-is-ccft.html#ccft-gettingstarted-IAM): 117 | 118 | ```json 119 | { 120 | "Version": "2012-10-17", 121 | "Statement": [ 122 | { 123 | "Effect": "Allow", 124 | "Action": "sustainability:GetCarbonFootprintSummary", 125 | "Resource": "*" 126 | } 127 | ] 128 | } 129 | ``` 130 | 131 | ### Q: What python packages do I need? 132 | 133 | You will need the python `requests` and `boto3` package. You can install it like this: 134 | 135 | ```bash 136 | python -m pip install requests boto3 137 | ``` 138 | 139 | ### Q: For what timeframe is data extracted? 140 | 141 | New carbon emissions data is available monthly, with a delay of three months as AWS gathers and processes the data that's required to provide your carbon emissions estimates. By default, the script extracts data starting from 39 months ago until three months before the current month. 142 | 143 | Example: When you are running the script in July 2023, the script extracts carbon emissions data from April 2020 to April 2023. (start_date: 2020-04-01, end_date: 2023-04-01) 144 | 145 | ### Q: How can I change the queried timeframe? 146 | 147 | Execute `python ccft_access.py -h` for help how the default interval can be changed. 148 | 149 | ```bash 150 | python ccft_access.py -h 151 | ``` 152 | 153 | ```bash 154 | usage: ccft_access.py [-h] [--start-date START_DATE] [--end-date END_DATE] 155 | 156 | Experimental retrieval of AWS Customer Carbon Footprint Tool console data. The data 157 | is queried for a closed interval from START_DATE to END_DATE (YYYY-MM-DD). The queried timeframe 158 | must be less than 36 months and not before 2020-01-01. 159 | 160 | optional arguments: 161 | -h, --help show this help message and exit 162 | --start-date START_DATE, -s START_DATE 163 | first month of the closed interval, default: 36 months before end month 164 | --end-date END_DATE, -e END_DATE 165 | last month of the closed interval, default: 3 months before current month 166 | ``` 167 | 168 | ### Q: How can I get the output prettyprinted? 169 | 170 | You can use `jq` to prettyprint the JSON output. [jq](https://jqlang.github.io/jq/) is a lightweight and flexible command-line JSON processor. If you use pip use `pip install jq` to install it. 171 | 172 | 173 | ```bash 174 | python ccft_access.py | jq . 175 | ``` 176 | 177 | ```bash 178 | { 179 | "accountId": "████████████", 180 | "query": { 181 | "queryDate": "2023-02-12", 182 | "startDate": "2020-01-01", 183 | "endDate": "2023-01-01" 184 | }, 185 | "emissions": { 186 | "carbonEmissionEntries": [ 187 | { 188 | "mbmCarbon": "0.048", 189 | "paceProductCode": "Other", 190 | "regionCode": "EMEA", 191 | "startDate": "2020-01-01" 192 | }, 193 | […] 194 | ``` 195 | 196 | ### Q: How do I get the data as a CSV? 197 | 198 | You can extend the use of `jq` in [the previous question](#q-how-can-i-get-the-output-prettyprinted) to transform the JSON output to a CSV file. 199 | 200 | ```bash 201 | python ccft_access.py | \ 202 | jq -r '{accountId} as $account | 203 | .emissions.carbonEmissionEntries | 204 | map(. + $account ) | 205 | (map(keys) | add | unique) as $cols | 206 | map(. as $row | $cols | map($row[.])) as $rows | 207 | $cols, $rows[] | @csv' > ccft-data.csv 208 | 209 | head ccft-data.csv 210 | ``` 211 | 212 | ```bash 213 | "accountId","mbmCarbon","paceProductCode","regionCode","startDate" 214 | "████████████","0.048","Other","EMEA","2020-01-01" 215 | […] 216 | ``` 217 | 218 | ## Multi-Account extraction FAQ 219 | 220 | ### Q: What does the application do on a high level? 221 | ![alt text](/static/ccft-sam_architecture.jpg) 222 | 223 | The application does the following on a high level: 224 | - On a monthly basis, an EventBridge schedule triggers a Step functions state machine 225 | - If it gets invoked for the first time, a data backfill happens which extracts AWS Customer Carbon Footprint Tool data from the past 40-4 months for all of the accounts in your AWS organization 226 | - The state machine checks if new monthly data is available (New data is available monthly, with a delay of three months) and retries daily if no new data is available yet 227 | - An AWS Lambda function executes the script to extract data from the AWS Customer Carbon Footprint Tool for all accounts of your AWS organization 228 | - The output is stored as a .json file in an S3 bucket 229 | - An Athena table and view is created, which you can directly use to query the data or visualize it with Amazon QuickSight 230 | 231 | ### Q: What resources am I deploying? 232 | 233 | This SAM template deploys the following resources: 234 | - Two S3 buckets: 235 | - `{AccountId}-{Region}-ccft-data` bucket where your carbon emissions data is being stored 236 | - `{AccountId}-{Region}-athenaresults` bucket where your Athena results are stored 237 | - Several Lambda functions with each a CloudWatch log group with retention time of 1 day and IAM roles with necessary permissions: 238 | - `get-account-ids.py` : returns all account ID's of an AWS organization, including the payer account ID 239 | - `check-first-invocation.py` : checks if a backfill of data is needed in case of first invocation 240 | - `extract-carbon-emissions-data.py` : executes the experimental programmatic access script for a given account ID and stores it in the `ccft-data` bucket as a .json file (example: `2023-04-01carbon_emissions.json`) 241 | - `create_alter_athena_view.py` : creates an Athena database (if not exists) and an Athena table (if not exists), as well as creates or updates two Athena views which point to the `ccft-data` bucket 242 | - An AWS Step Function State Machine `ExtractCarbonEmissionsStateMachine`. You can find the definition in statemachine/extract_carbon_emissions.asl.json 243 | - An EventBridge scheduler `ExtractCarbonEmissionsFunctionScheduleEvent` as a trigger for the AWS Step Functions state machine which runs at the 15th day of every month: cron(0 0 15 * ? *) 244 | - An IAM role `ccft-sam-script-ExtractCarbonEmissionsFunctionSche-{id}` for the EventBridge scheduler 245 | 246 | You can find details on the resources that are created within the [`template.yaml`](./MultiAccountApplication/template.yaml) file. 247 | 248 | ### Q: What does the state machine do? 249 | ![alt text](/static/stepfunctions_graph.svg) 250 | 251 | - **GetAccountIDs**: A Lambda function extracts all account ID's of the organization including the payer account ID. It calculates the timeframes for the backfill and the retrieval of latest CCFT data. 252 | 253 | - **CheckFirstInvocation**: The statemachine checks if it is invoked for the first time by checking if the created S3 bucket `{AccountId}-{Region}-ccft-data` is empty. If yes, it continues to (3). If there are already objects in the bucket (which means it is not the first invocation), it directly jumps to **ExtractCarbonEmissions-Canary**. 254 | 255 | - **ProcessAccounts-Backfill**: The first invocation triggers a backfill of data for the past 40-4months. 256 | For every account that belongs to the organization you are running this in a Lambda function with the ccft script is triggered, which extracts the AWS Customer Carbon Footprint Tool data for the past 40 months to the past 4 months and stores it as one .json file per account in the ccft-data bucket, since the maximum duration that we can extract carbon emissions data from is 36 months. (Example: If the first invocation happens in August 2023, the script will extract data from April 2020 to April 2023). This State Machine Map state is set to 50 concurrent executions. 257 | 258 | - **ExtractCarbonEmissions-Canary**/ **Choice: New Data Available**: As the publishing date for new monthly CCFT data can vary, the state machine first checks if there is data for a canary account (by default the payer) available (for three months ago). If not, it goes to a waiting state (**RetryLambdaDaily**). If there is data available, it goes forward to **ProcessAccounts**. 259 | 260 | - **RetryLambdaDaily**: If new monthly data is not yet available, the statemachine waits for one day and then tries (4) again. 261 | 262 | - **ProcessAccounts**: If new monthly data is available, we can continue to extract the data for all accounts of an AWS organization. The state machine extracts triggers a Lambda function with the ccft-script to extract CCFT data for all account ID's, and stores the data within one .json file. This State Machine Map state is set to 50 concurrent executions. 263 | 264 | - **CreateAthenaTableView**: If they don't exist, the Athena database, table, and views are created that point to the ccft-data bucket. 265 | 266 | ### Q: How can I deploy the application? 267 | 268 | You can deploy the application via the Serverless Application Repository **or** with the SAM CLI. 269 | 270 | #### Option 1: Deployment via the AWS Serverless Application Repository 271 | 272 | The [AWS Serverless Application Repository](https://aws.amazon.com/serverless/serverlessrepo/) is a managed repository for serverless applications. Using the Serverless Application Repository (SAR), you don't need to clone, build, package, or publish source code to AWS before deploying it. To deploy the application, go to the [Experimental Programmatic Access application](https://serverlessrepo.aws.amazon.com/applications/eu-central-1/406252154896/experimental-programmatic-access-ccft). 273 | 274 | [![cloudformation-launch-button](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://serverlessrepo.aws.amazon.com/applications/eu-central-1/406252154896/experimental-programmatic-access-ccft) 275 | 276 | 277 | In the AWS Management console, you can view the application's permissions and resources, and configure the application in the `Application settings` section. 278 | * Select `Deploy` to deploy the application. 279 | * Navigate to the CloudFormation dashboard by selecting `Deployments` and `CloudFormation stack`. Here, you can see the stack that was just deployed. You can navigate to the `Resources` tab to see all resources that were created as part of this stack. 280 | * The state machine will automatically be triggered on the next 15th. If you want to run the application already now, you can also navigate to your `ExtractCarbonEmissionsStateMachine` Step Functions State Machine. Select **Start execution**. You can leave everything as is, and select **Start execution**. 281 | 282 | #### Option 2: Deployment with the SAM CLI 283 | 284 | The Serverless Application Model Command Line Interface (SAM CLI) is an extension of the AWS CLI that adds functionality for building and testing Lambda applications. 285 | 286 | To use the SAM CLI, you need the following tools. 287 | 288 | * SAM CLI - [Install the SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) 289 | * [Python 3 installed](https://www.python.org/downloads/) 290 | * Docker - [Install Docker community edition](https://docs.docker.com/get-docker/) 291 | 292 | (1) Clone the repository. 293 | 294 | (2) Navigate to the cloned repository and the folder `MultiAccountApplication`, so that you are in the folder that includes the [`template.yaml`](./MultiAccountApplication/template.yaml) file. 295 | To build and deploy your application for the first time, run the following commands: 296 | 297 | ```bash 298 | sam build 299 | sam deploy --guided --profile 300 | ``` 301 | 302 | The first command will build the source of your application. If you get an error message related to Python 3.11 runtime dependencies, you can also use `sam build --use-container` to build your serverless application's code in a Docker container. 303 | 304 | The second command will package and deploy your application to AWS, with a series of prompts: 305 | 306 | * **Stack Name**: The name of the stack to deploy to CloudFormation. This should be unique to your account and region. You can leave the default value of `ccft-sam-script`. 307 | * **AWS Region**: The AWS region you want to deploy your app to. 308 | * **CarbonEmissionsDataBucketName** parameter: Name suffix (w/o prefixes for acct ID, region) for carbon emissions data bucket. 309 | * **CarbonEmissionsDataFileName** parameter: Name prefix for the .json file where carbon emissions data is stored. 310 | * **CCFTRoleName** parameter: Name of the IAM role that was deployed into all member accounts and gives read access to the AWS CCFT data. 311 | * **GlueDatabaseName** parameter: Name of the Glue database used for Amazon Athena. 312 | * **Confirm changes before deploy**: If set to yes, any change sets will be shown to you before execution for manual review. If set to no, the AWS SAM CLI will automatically deploy application changes. 313 | * **Allow SAM CLI IAM role creation**: This SAM template creates AWS IAM roles required for the AWS Lambda function(s) included to access AWS services. By default, these are scoped down to minimum required permissions. To deploy an AWS CloudFormation stack which creates or modifies IAM roles, the `CAPABILITY_IAM` value for `capabilities` must be provided. If permission isn't provided through this prompt, to deploy this example you must explicitly pass `--capabilities CAPABILITY_IAM` to the `sam deploy` command. 314 | * **Save arguments to samconfig.toml**: If set to yes, your choices will be saved to a configuration file inside the project, so that in the future you can just re-run `sam deploy` without parameters to deploy changes to your application. 315 | 316 | Confirm changeset to be deployed and wait until all resources are deployed. You will see a success message such as `Successfully created/updated stack - ccft-sam-script in eu-central-1`. 317 | 318 | (3) Within the AWS console, navigate to the CloudFormation dashboard. Here, you will see the stack that was just deployed. You can navigate to the `Resources` tab to see all resources that were created as part of this stack. 319 | 320 | (4) The state machine will automatically be triggered on the next 15th. If you want to run the application already now, you can also navigate to your `ExtractCarbonEmissionsStateMachine` Step Functions State Machine. Select **Start execution**. You can leave everything as is, and select **Start execution**. 321 | 322 | ### Q: Into which account should I deploy this application? 323 | 324 | You can run this from any account within your organization, as long as you set up the necessary permissions. 325 | 326 | ### Q: What permissions are needed to run this? 327 | 328 | In order to successfully extract carbon emissions data from the central account for all child accounts, follow these steps: 329 | 330 | (1) Deploy an IAM role named `ccft-read-role` with the following AWS IAM policy that contains the [AWS Customer Carbon Footprint Tool IAM permission](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/what-is-ccft.html#ccft-gettingstarted-IAM) into all child accounts. To do this for all accounts of your AWS organizations, there are several options which are explained in [Q: How can I deploy IAM roles into multiple accounts](#q-how-can-i-deploy-iam-roles-into-multiple-aws-accounts) 331 | 332 | ```json 333 | { 334 | "Version": "2012-10-17", 335 | "Statement": [ 336 | { 337 | "Effect": "Allow", 338 | "Action": "sustainability:GetCarbonFootprintSummary", 339 | "Resource": "*" 340 | } 341 | ] 342 | } 343 | ``` 344 | 345 | (2) Additionally, you need to set-up a trust relationship so that the Lambda function `extract-carbon-emissions-data` in the central account (where you've deployed the SAM application) can assume this role. Update {Account} with the account ID of the account where you've deployed this application. 346 | 347 | ```json 348 | { 349 | "Version": "2012-10-17", 350 | "Statement": [ 351 | { 352 | "Effect": "Allow", 353 | "Principal": { 354 | "AWS": "arn:aws:iam::{Account}:role/extract-emissions-lambda-role" 355 | }, 356 | "Action": "sts:AssumeRole", 357 | "Condition": {} 358 | } 359 | ] 360 | } 361 | ``` 362 | 363 | (3) Optional: if you have given the IAM role a different name, you can change the parameter **CCFTRoleName** when deploying the SAM application. Make sure that all roles within all child accounts have the same name. 364 | 365 | (4) The `get-account-ids.py` Lambda function calls the AWS Organization's ListAccounts API. This operation can be called only from the organization's management account or by a member account that is a delegated administrator for an AWS service. You can set up a [delegation policy](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_delegate_policies.html?icmpid=docs_orgs_console) which allows the account where you're deploying this solution to call the ListAccounts API. 366 | 367 | ### Q: How can I deploy IAM roles into multiple AWS accounts? 368 | 369 | Depending on your AWS Organization set-up, there are several ways to achieve having the same IAM role with a trust relationship deployed into all accounts of your AWS organization. 370 | 371 | (1) In an AWS Control Tower environment, use the [Customization for Control Tower](https://aws.amazon.com/solutions/implementations/customizations-for-aws-control-tower/) (CfCT) CI/CD pipeline to deploy the CCFT read-only IAM role for existing and new accounts. A sample manifest and CFn template are included in the [`ccft-org-read-role`](./MultiAccountApplication/ccft-org-read-role/) folder in this repository. 372 | 373 | (2) Using [AWS CloudFormation StackSets](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/what-is-cfnstacksets.html) to deploy IAM roles to existing and future accounts. Check out the blog post [Use AWS CloudFormation StackSets for Multiple Accounts in an AWS Organization](https://aws.amazon.com/blogs/aws/new-use-aws-cloudformation-stacksets-for-multiple-accounts-in-an-aws-organization/) for more details. A sample template (`ccft-read-only.yaml`) for the role is included in the [`ccft-org-read-role/ccft-role`](./MultiAccountApplication/ccft-org-read-role/ccft-role/) folder in this repository. 374 | 375 | ### Q: Can I change the queried timeframe? 376 | The application extracts data for three months past the month it is running. Example: The application extracts data for April 2023 when it runs in July 2023. 377 | 378 | You can override the timeframe when you manually start the Step Functions workflow. 379 | - In the Step Functions console, click on `New Execution` 380 | - Enter the following input and change "YYYY-MM-DD". Remember that the application extracts data for three months past the override date. 381 | ``` 382 | { 383 | "override_today": "YYYY-MM-DD" 384 | } 385 | ``` 386 | 387 | ### Q: What can I do with the data? 388 | As a result of a successful run through the state machine, new emissions data from the AWS Customer Carbon Footprint Tool will be available monthly in the S3 bucket `{AccountId}-{Region}-ccft-data` in .json format. 389 | 390 | To make it easier to use this data, the state machine also creates an Athena table and two views to directly query the data with SQL. Navigate to the Amazon Athena console and select the database **carbon_emissions**. You should see a table named `carbon_emissions_table` and two views called `carbon_emissions_view` and `carbon_emissions_aggregate_view`. 391 | 392 | `carbon_emissions_view` shows the monthly carbon emissions per account ID, product code and region code. `carbon_emissions_aggregate_view` shows the aggregated carbon emissions per account ID per month, and the respective entries for gridmixinefficiency and servermedianinefficiency. 393 | 394 | ![Athena Console Screenshot](/static/athena_table_view.png) 395 | 396 | If you select **Preview Table** using the three dots next to your table, you can see the nested json data with one row per json file. You can also view the statement used to create the table by selecting **Generate table DDL** using the three dots next to your table. 397 | 398 | Next, select **Preview view** using the three dots next to your view. When you select **Show/edit query** you can also see and modify the query to create the view. The view includes the unnested data, with one row per account and month data. You can use SQL statements to directly query the data. If you want to find all emissions for a specific month, you can for example use the following statement: 399 | ```sql 400 | SELECT * FROM carbon_emissions_view WHERE startdate = '2022-01-01'; 401 | ``` 402 | 403 | If you want to visualize the data, you can do so by using Amazon QuickSight. Check out the following documentation entry to understand [how you can create a QuickSight dataset using Amazon Athena data](https://docs.aws.amazon.com/quicksight/latest/user/create-a-data-set-athena.html). 404 | 405 | ### Other things to consider 406 | - If you're logged in to a management account of AWS Organizations, the customer carbon footprint tool reports an aggregate of the member account data for the duration that those accounts were a part of your management account. If you're logged in to a member account, the customer carbon footprint tool reports emission data for all the periods for this account. This is regardless of any changes that might have occurred to your account's associated membership in one of the AWS Organizations. 407 | - If data isn't available for your account, your account might be too new to show data. After each month, you might have a delay of up to three months for AWS to show your carbon emission estimates. 408 | - If your AWS Customer Carbon Footprint Tool emissions are zero, the script will also return `0.0`. Please note, that you will not see the product split or region split in this case (`paceProductCode` and `regionCode` under `carbonEmissionEntries` will not be returned). In your Athena view, `paceproductcode` and `regioncode` will be empty if the emissions of an account for a specific month are `0`. 409 | - `servermedianinefficiency` and `gridmixinefficiency` data are provided per month per accountID, not in the granularity of `paceproductcode` and `regioncode` 410 | 411 | ### Troubleshooting 412 | - By default, the retention period for the Lambda function log groups are set to 1 day. You can change the `RetentionInDays` parameter in the [SAM template](./MultiAccountApplication/template.yaml). 413 | - For further trouble shooting, enable [logging on the state machine](https://docs.aws.amazon.com/step-functions/latest/dg/cw-logs.html). 414 | - If you run into `PermissionDenied` errors, check that the correct permissions are set up. Additionally, check for [Service Control Policies (SCPs)](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html) that further limit access centrally and trouble-shoot using [IAM policy simulator](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_testing-policies.html). 415 | 416 | ### What are costs for running this application? 417 | The cost for running this application depends on the number of accounts that are part of your organization. You can use this [AWS Pricing Calculator example](https://calculator.aws/#/estimate?id=1963803b68ed07e959eb8cbd2b7b3a7059bbfbfb) and adapt it to your requirements. There are no upfront cost to running this application; you pay for the resources that are created and used as part of the application. Major services used are the following: 418 | 419 | - [AWS Lambda](https://docs.aws.amazon.com/whitepapers/latest/how-aws-pricing-works/lambda.htmlaw): 420 | - Example: If you have 100 accounts as part of your AWS organization, the first invocation (with backfill) will result in 205 Lambda function invocations, after the backfill 104 Lambda invocations per month (given that new AWS CCFT data is available on the 15th of the month). The `extract-carbon-emissions-data.py` and `backfill-data.py` Lambda functions run approximately 6-10 seconds per account ID. 421 | - [AWS Step Functions](https://aws.amazon.com/step-functions/pricing/) 422 | - Example: The state machine will run once monthly, in the case of the backfill with 213 state transitions per month, or 110 state transitions after the backfill (given that new AWS CCFT data is available on the 15th of the month and the state machine run is successful). 423 | - [Amazon S3](https://aws.amazon.com/s3/pricing/) 424 | - Example: The size of the backfill .json file is ~10KB, and the size of a .json file for one month's data ~1KB (note that this depends on the specific data you get back). Running the application for 100 AWS accounts, this would result in a total storage capacity for the first month of 1.1MB, with an additional 0.1MB per month. 425 | - [Amazon Athena](https://aws.amazon.com/athena/pricing/) 426 | - Example: You run 10 queries a month and scan 100MB of data per query (note that this depends on the number of accounts of your AWS organization). 427 | 428 | ### Q: How can I specify the account ids to retrieve data from? 429 | 430 | By default the application retrieves data from all accounts of the AWS Organization with the payer as the canary account. If you want to override the account ids you can add them to the scheduling event in the template: 431 | 432 | ``` 433 | ComplexScheduleEvent: 434 | Type: ScheduleV2 435 | Properties: 436 | ScheduleExpression: "cron(0 0 15 * ? *)" 437 | Input: "{\"override_accounts\": [\"YOUR-ACCOUNT-ID\", \"YOUR-OTHER-ID\"]}" # add this line 438 | FlexibleTimeWindow: 439 | Mode: FLEXIBLE 440 | MaximumWindowInMinutes: 60 441 | ``` 442 | 443 | The first account in the list will be used as the canary account. 444 | 445 | ### Q: Cleanup - How can I delete the application? 446 | 447 | To delete the sample application that you created, you can use the AWS CLI. Make sure to empty the buckets that were created as part of this application before you run the following command. Assuming you used `ccft-sam-script` for the stack name, you can run the following: 448 | 449 | ```bash 450 | aws cloudformation delete-stack --stack-name ccft-sam-script 451 | ``` 452 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "experimental-programmatic-access-ccft", 3 | "version": "1.2.0", 4 | "description": "This is a serverless application that automates the monthly extraction of new CCFT data within a multi account structure.", 5 | "author": { 6 | "name": "experimental-programmatic-access-ccft-team" 7 | }, 8 | "contributors": [ 9 | { 10 | "name" : "Steffen Grunwald" 11 | }, 12 | { 13 | "name" : "Katja Philipp" 14 | } 15 | ], 16 | "license": "MIT-0", 17 | "homepage": "https://github.com/aws-samples/experimental-programmatic-access-ccft/MultiAccountApplication" 18 | } -------------------------------------------------------------------------------- /static/athena_table_view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/experimental-programmatic-access-ccft/d1c273542acee52adedfaf85a44d48d865929a95/static/athena_table_view.png -------------------------------------------------------------------------------- /static/ccft-api-explanation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/experimental-programmatic-access-ccft/d1c273542acee52adedfaf85a44d48d865929a95/static/ccft-api-explanation.png -------------------------------------------------------------------------------- /static/ccft-sam_architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/experimental-programmatic-access-ccft/d1c273542acee52adedfaf85a44d48d865929a95/static/ccft-sam_architecture.jpg -------------------------------------------------------------------------------- /static/stepfunctions_graph.svg: -------------------------------------------------------------------------------- 1 | 2 | Start End CreateAthenaTableView GetAccountIDs CheckFirstInvocation Choice: First Invocation ExtractCarbonEmissions-Backfill HandleError-Process Accounts1 ProcessAccounts-Backfill ExtractCarbonEmissions-Canary Choice: New Data Available HandleError ExtractCarbonEmissions-All HandleError-Process Accounts ProcessAccounts RetryLambdaDaily Parallel --------------------------------------------------------------------------------