├── tests ├── __init__.py ├── starbase │ └── __init__.py ├── account_index │ ├── __init__.py │ ├── testing_plugins │ │ ├── __init__.py │ │ ├── bad_plugin │ │ │ └── __init__.py │ │ └── basic_plugin │ │ │ └── __init__.py │ ├── conftest.py │ └── test_account_index_components.py ├── worker_ship_utils │ ├── __init__.py │ ├── testing_plugins │ │ ├── __init__.py │ │ ├── bad_plugin │ │ │ └── __init__.py │ │ └── basic_plugin │ │ │ └── __init__.py │ ├── sample_payloads │ │ ├── empty_payload.yaml │ │ ├── invalid_base.yaml │ │ ├── invalid_yaml.yaml │ │ ├── sample_payload.yaml │ │ ├── sample_account_payload.yaml │ │ └── sample_account_region_payload.yaml │ └── conftest.py ├── starfleet_included_plugins │ ├── __init__.py │ ├── iam │ │ ├── __init__.py │ │ ├── bad_payloads │ │ │ ├── bad_role_iambic.yaml │ │ │ └── bad_role_marshmallow.yaml │ │ ├── test_role_payload.yaml │ │ └── conftest.py │ ├── aws_config │ │ ├── __init__.py │ │ ├── test_payload.yaml │ │ └── conftest.py │ ├── github_sync │ │ ├── __init__.py │ │ ├── test_files │ │ │ ├── file_one.txt │ │ │ ├── nested_dir │ │ │ │ ├── some_other_file.json │ │ │ │ └── last_file.yaml │ │ │ └── file_two.yaml │ │ ├── test_payload.yaml │ │ ├── test_schemas.py │ │ └── test_auth.py │ └── account_index_generator │ │ ├── __init__.py │ │ └── test_payload.yaml ├── bad_configuration_files │ └── bad_configuration.yaml ├── test_configuration_files │ ├── another_file.yaml │ ├── github_sync.yaml │ ├── aws_config.yaml │ ├── iam_role_worker.yaml │ ├── account_index_generator.yaml │ ├── test_base_configuration.yaml │ └── basic_plugin.yaml ├── test_cli_components.py └── test_starfleet.py ├── src ├── starfleet │ ├── __init__.py │ ├── cli │ │ ├── __init__.py │ │ ├── entrypoint.py │ │ └── components.py │ ├── utils │ │ ├── __init__.py │ │ ├── niceties.py │ │ ├── logging.py │ │ ├── secrets.py │ │ ├── plugin_loader.py │ │ ├── configuration.py │ │ └── config_schema.py │ ├── starbase │ │ ├── __init__.py │ │ └── entrypoints.py │ ├── worker_ships │ │ ├── __init__.py │ │ ├── plugins │ │ │ ├── __init__.py │ │ │ ├── account_index_generator │ │ │ │ └── __init__.py │ │ │ ├── github_sync │ │ │ │ └── __init__.py │ │ │ ├── aws_config │ │ │ │ └── __init__.py │ │ │ └── iam │ │ │ │ ├── __init__.py │ │ │ │ └── iambic_imports.py │ │ ├── niceties.py │ │ ├── lambda_utils.py │ │ └── loader.py │ ├── account_index │ │ ├── __init__.py │ │ ├── plugins │ │ │ ├── __init__.py │ │ │ └── starfleet_default_index │ │ │ │ └── __init__.py │ │ ├── loader.py │ │ └── schematics.py │ ├── startup.py │ └── configuration_files │ │ └── configuration.yaml └── requirements.txt ├── mkdocs ├── images │ ├── S3Sync.png │ ├── SlackToken.png │ ├── ConfigSuccess.png │ ├── GitHubAppMenu.png │ ├── CertFindReplace.png │ ├── SlackChannelId.png │ ├── SlackCreateApp.png │ ├── SlackAddToChannel.png │ ├── SlackOAuthScopes.png │ ├── GItHubInstallationId.png │ ├── GitHubInstallAppMenu.png │ ├── GitHubInstallAppPage.png │ ├── GitHub3rdPartyAppsMenu.png │ ├── SlackAppNameWorkspace.png │ └── SlackOAuthAndPermissions.png ├── logo_files │ ├── Logo.png │ ├── chmmr.png │ ├── urquan.png │ └── chenjesu.png ├── blog │ ├── .authors.yml │ ├── index.md │ └── posts │ │ ├── fwd-cloudsec.md │ │ ├── ecr.md │ │ └── iam-role-worker.md ├── architecture │ ├── Alerts.md │ ├── CLI.md │ ├── Secrets.md │ ├── ResidentIAMRole.md │ ├── AccountIndex.md │ └── Overview.md ├── stylesheets │ └── extra.css ├── Links.md ├── userGuide │ ├── PayloadTemplates.md │ ├── awsConfig │ │ ├── CLI.md │ │ └── AWSConfigWorker.md │ ├── AccountIndexGenerator.md │ ├── IAM │ │ └── Roles │ │ │ └── CLI.md │ ├── Starbase.md │ ├── GitHubSync │ │ ├── Overview.md │ │ ├── Template.md │ │ └── CLI.md │ ├── Overview.md │ └── Secrets.md ├── Attributions.md ├── developerGuide │ ├── primaryComponents │ │ ├── Loggers.md │ │ ├── SecretsManager.md │ │ ├── workerShips │ │ │ ├── CLI.md │ │ │ ├── LambdaEntrypoints.md │ │ │ └── Loader.md │ │ ├── ConfigurationManager.md │ │ └── AccountIndexer.md │ ├── GeneratingDocs.md │ └── CheckList.md ├── installation │ ├── WrapUp.md │ ├── PrepareConfiguration.md │ ├── PreparePayload.md │ ├── Overview.md │ └── SetupECR.md └── index.md ├── pytest.ini ├── setup.py ├── ATTRIBUTIONS ├── .github ├── workflows │ └── python.yml └── dependabot.yml ├── README.md ├── tox.ini ├── pyproject.toml ├── Dockerfile └── sample_samconfig.toml /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/starfleet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/starbase/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/starfleet/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/starfleet/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/account_index/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/starfleet/starbase/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/starfleet/account_index/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/starfleet/account_index/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/account_index/testing_plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/iam/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/testing_plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/aws_config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/github_sync/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/account_index_generator/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/sample_payloads/empty_payload.yaml: -------------------------------------------------------------------------------- 1 | # File is intentionally blank 2 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/sample_payloads/invalid_base.yaml: -------------------------------------------------------------------------------- 1 | MissingRequiredFields: True 2 | -------------------------------------------------------------------------------- /tests/bad_configuration_files/bad_configuration.yaml: -------------------------------------------------------------------------------- 1 | LOL: 2 | Where: is 3 | Starfleet: ?? 4 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/github_sync/test_files/file_one.txt: -------------------------------------------------------------------------------- 1 | This is just a file... 2 | -------------------------------------------------------------------------------- /mkdocs/images/S3Sync.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/S3Sync.png -------------------------------------------------------------------------------- /mkdocs/logo_files/Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/logo_files/Logo.png -------------------------------------------------------------------------------- /tests/test_configuration_files/another_file.yaml: -------------------------------------------------------------------------------- 1 | SOMEOTHER: 2 | TestFile: has been loaded properly 3 | -------------------------------------------------------------------------------- /mkdocs/images/SlackToken.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/SlackToken.png -------------------------------------------------------------------------------- /mkdocs/logo_files/chmmr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/logo_files/chmmr.png -------------------------------------------------------------------------------- /mkdocs/logo_files/urquan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/logo_files/urquan.png -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/github_sync/test_files/nested_dir/some_other_file.json: -------------------------------------------------------------------------------- 1 | {"some": "otherfile"} 2 | -------------------------------------------------------------------------------- /mkdocs/images/ConfigSuccess.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/ConfigSuccess.png -------------------------------------------------------------------------------- /mkdocs/images/GitHubAppMenu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/GitHubAppMenu.png -------------------------------------------------------------------------------- /mkdocs/logo_files/chenjesu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/logo_files/chenjesu.png -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/github_sync/test_files/nested_dir/last_file.yaml: -------------------------------------------------------------------------------- 1 | The: 2 | last: 3 | - file 4 | -------------------------------------------------------------------------------- /mkdocs/images/CertFindReplace.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/CertFindReplace.png -------------------------------------------------------------------------------- /mkdocs/images/SlackChannelId.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/SlackChannelId.png -------------------------------------------------------------------------------- /mkdocs/images/SlackCreateApp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/SlackCreateApp.png -------------------------------------------------------------------------------- /mkdocs/images/SlackAddToChannel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/SlackAddToChannel.png -------------------------------------------------------------------------------- /mkdocs/images/SlackOAuthScopes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/SlackOAuthScopes.png -------------------------------------------------------------------------------- /mkdocs/images/GItHubInstallationId.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/GItHubInstallationId.png -------------------------------------------------------------------------------- /mkdocs/images/GitHubInstallAppMenu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/GitHubInstallAppMenu.png -------------------------------------------------------------------------------- /mkdocs/images/GitHubInstallAppPage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/GitHubInstallAppPage.png -------------------------------------------------------------------------------- /mkdocs/images/GitHub3rdPartyAppsMenu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/GitHub3rdPartyAppsMenu.png -------------------------------------------------------------------------------- /mkdocs/images/SlackAppNameWorkspace.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/SlackAppNameWorkspace.png -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/github_sync/test_files/file_two.yaml: -------------------------------------------------------------------------------- 1 | This: 2 | is: 3 | Just: 4 | - a 5 | - file 6 | -------------------------------------------------------------------------------- /mkdocs/images/SlackOAuthAndPermissions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gemini-oss/starfleet/HEAD/mkdocs/images/SlackOAuthAndPermissions.png -------------------------------------------------------------------------------- /tests/worker_ship_utils/sample_payloads/invalid_yaml.yaml: -------------------------------------------------------------------------------- 1 | [][][][][][][{}{}{}{}{}{\\\\[p[p]]}]This { 2 | Is 3 | } 4 | \\ Not 5 | A // 6 | Yaml 7 | ;; File 8 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | ; The option below will break click unit tests: https://github.com/pallets/click/issues/824 3 | log_cli = false 4 | usefixtures = mock_retry 5 | -------------------------------------------------------------------------------- /mkdocs/blog/.authors.yml: -------------------------------------------------------------------------------- 1 | authors: 2 | mikegrima: 3 | name: Mike Grima 4 | description: Creator of Starfleet 5 | avatar: https://github.com/mikegrima.png 6 | -------------------------------------------------------------------------------- /mkdocs/blog/index.md: -------------------------------------------------------------------------------- 1 | # The Starfleet Blog 2 | 3 | This is the main go-to for announcements on the Starfleet project. Follow this for updates on new features, bug fixes, etc. 4 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/sample_payloads/sample_payload.yaml: -------------------------------------------------------------------------------- 1 | TemplateName: SampleBasePayloadTemplate 2 | TemplateDescription: This is a template for unit test purposes only. 3 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/sample_payloads/sample_account_payload.yaml: -------------------------------------------------------------------------------- 1 | TemplateName: SampleAccountRegionTemplate 2 | TemplateDescription: This is a template for unit test purposes only for ACCOUNT_REGION workers. 3 | IncludeAccounts: 4 | AllAccounts: True 5 | -------------------------------------------------------------------------------- /mkdocs/architecture/Alerts.md: -------------------------------------------------------------------------------- 1 | # Alerts & Notifications 2 | 3 | Starfleet supports emitting alerts and notifications to external systems. As of today, we include a Slack plugin that allows you to emit notifications to Slack. More details on this is in the User and Developer Guides. 4 | -------------------------------------------------------------------------------- /tests/test_configuration_files/github_sync.yaml: -------------------------------------------------------------------------------- 1 | GitHubSyncWorkerShip: 2 | Enabled: True 3 | TemplatePrefix: github_sync/ 4 | InvocationQueueUrl: https://sqs.amazonaws.com/replace-me 5 | InvocationSources: 6 | - EVENTBRIDGE_TIMED_EVENT 7 | EventBridgeTimedFrequency: HOURLY 8 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/account_index_generator/test_payload.yaml: -------------------------------------------------------------------------------- 1 | TemplateName: AccountIndexGeneratorShip 2 | TemplateDescription: This is a template for Account Index Generator Worker ship 3 | AccountInventoryBucket: account-inventory-s3-bucket 4 | InventoryBucketRegion: us-east-2 5 | -------------------------------------------------------------------------------- /mkdocs/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | /* Make the content area expand to 88% of the width */ 2 | .md-grid { 3 | max-width: 88%; 4 | } 5 | 6 | 7 | .md-typeset h1 { 8 | font-weight: 700; 9 | } 10 | 11 | .md-typeset h2 { 12 | font-weight: 500; 13 | } 14 | 15 | .md-header { 16 | background-image: url(); 17 | } -------------------------------------------------------------------------------- /src/requirements.txt: -------------------------------------------------------------------------------- 1 | iambic-core==0.11.97 2 | cloudaux-lite==1.0.0 3 | click==8.1.7 4 | PyYAML==6.0.1 5 | marshmallow==3.21.1 6 | retry==0.9.2 7 | pyjwt==2.8.0 8 | requests==2.31.0 9 | # cryptography==42.0.5 Handled by iambic-core 10 | botocore==1.34.74 11 | boto3==1.34.74 12 | slack-sdk==3.27.1 13 | deepdiff==6.7.1 14 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/sample_payloads/sample_account_region_payload.yaml: -------------------------------------------------------------------------------- 1 | TemplateName: SampleAccountRegionTemplate 2 | TemplateDescription: This is a template for unit test purposes only for ACCOUNT_REGION workers. 3 | IncludeAccounts: 4 | AllAccounts: True 5 | IncludeRegions: 6 | - ALL 7 | ExcludeRegions: 8 | - us-west-1 9 | -------------------------------------------------------------------------------- /tests/test_configuration_files/aws_config.yaml: -------------------------------------------------------------------------------- 1 | AwsConfigWorkerShip: 2 | Enabled: True 3 | TemplatePrefix: awsconfig/ 4 | InvocationQueueUrl: https://sqs.amazonaws.com/replace-me 5 | InvocationSources: 6 | - EVENTBRIDGE_TIMED_EVENT 7 | EventBridgeTimedFrequency: SIX_HOURLY 8 | WorkerRoleToAssume: starfleet-worker-basic-test-role 9 | -------------------------------------------------------------------------------- /tests/test_configuration_files/iam_role_worker.yaml: -------------------------------------------------------------------------------- 1 | IamRoleWorkerShip: 2 | Enabled: True 3 | TemplatePrefix: IAM/Roles/ 4 | InvocationQueueUrl: https://sqs.amazonaws.com/replace-me 5 | InvocationSources: 6 | - EVENTBRIDGE_TIMED_EVENT 7 | EventBridgeTimedFrequency: THIRTY_MIN 8 | WorkerRoleToAssume: starfleet-worker-basic-test-role 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Main setup file for starfleet 2 | Setup file for the starfleet package 3 | 4 | This project is using the modern pyproject.toml format and this file is only required for running: 5 | ``pip install -e .`` 6 | 7 | :Module: setup 8 | :Author: Mike Grima 9 | """ 10 | 11 | from setuptools import setup 12 | 13 | setup() 14 | -------------------------------------------------------------------------------- /mkdocs/Links.md: -------------------------------------------------------------------------------- 1 | # Links and other resources 2 | 3 | ## Attributions 4 | See [the attributions](Attributions.md) page for additional attributions for this OSS project. 5 | 6 | ## GitHub 7 | 1. Main repository link: [https://github.com/gemini-oss/starfleet](https://github.com/gemini-oss/starfleet) 8 | 1. Other cool Gemini OSS projects: [https://github.com/gemini-oss](https://github.com/gemini-oss) 9 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/github_sync/test_payload.yaml: -------------------------------------------------------------------------------- 1 | TemplateName: TestGitHubSync 2 | TemplateDescription: Tests out the GitHub syncing worker 3 | Organization: fakeorg 4 | Repository: myrepo 5 | BranchName: main 6 | GitHubAppId: "1234567" 7 | GitHubInstallationId: "987654" 8 | BucketName: some-bucket 9 | BucketRegion: us-east-2 10 | ExtractZipContents: True 11 | DeleteMissingFiles: True 12 | -------------------------------------------------------------------------------- /mkdocs/architecture/CLI.md: -------------------------------------------------------------------------------- 1 | # Command Line Interface (CLI) 2 | 3 | This will be covered more in the User and Developer Guides, but is mentioned here that Starfleet worker ships expose a CLI so that they can be invoked locally. This is useful for debugging issues or if you just need to run a given payload in one AWS account. 4 | 5 | The CLIs are based on the excellent Python [Click](https://click.palletsprojects.com/) framework. 6 | -------------------------------------------------------------------------------- /tests/test_configuration_files/account_index_generator.yaml: -------------------------------------------------------------------------------- 1 | AccountIndexGeneratorShip: 2 | Enabled: True 3 | TemplatePrefix: sample_worker/ 4 | InvocationQueueUrl: https://sqs.amazonaws.com/replace-me 5 | InvocationSources: 6 | - EVENTBRIDGE_TIMED_EVENT 7 | EventBridgeTimedFrequency: HOURLY 8 | OrgAccountAssumeRole: starfleet-worker-basic-test-role 9 | OrgAccountId: "123456789012" 10 | OrgRootId: r-123456 11 | DescribeRegionsAssumeRole: starfleet-worker-basic-test-role 12 | -------------------------------------------------------------------------------- /mkdocs/architecture/Secrets.md: -------------------------------------------------------------------------------- 1 | # Secrets Management 2 | 3 | Starfleet has an internal component for managing secrets that may be used throughout the application. This makes use of [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/) to securely store sensitive data, like tokens, keys, and passwords that a worker may need to utilize. 4 | 5 | If making use of Slack alerts, then the Slack token will need to be stored here. 6 | 7 | More details on the secrets management is in the User and Developer guides. 8 | -------------------------------------------------------------------------------- /tests/account_index/testing_plugins/bad_plugin/__init__.py: -------------------------------------------------------------------------------- 1 | """A sample BAD account index plugin for use in unit testing exceptions. 2 | 3 | This tests that the exceptions logic is working properly. 4 | 5 | :Module: starfleet.tests.account_index.testing_plugins.bad_plugin 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | # This file is intentionally left blank 12 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/testing_plugins/bad_plugin/__init__.py: -------------------------------------------------------------------------------- 1 | """A sample BAD worker plugin for use in unit testing exceptions. 2 | 3 | This tests that the exceptions logic is working properly. 4 | 5 | :Module: starfleet.tests.worker_ship_utils.testing_plugins.bad_plugin 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | # This file is intentionally left blank 12 | -------------------------------------------------------------------------------- /tests/test_configuration_files/test_base_configuration.yaml: -------------------------------------------------------------------------------- 1 | STARFLEET: 2 | DeploymentRegion: us-east-2 3 | TemplateBucket: template-bucket 4 | FanOutQueueUrl: https://sqs.amazonaws.com/fanout-queue 5 | AccountIndex: TestingAccountIndexPlugin 6 | LogLevel: DEBUG 7 | SecretsManager: 8 | SecretId: starfleet-secrets 9 | SecretRegion: us-east-2 10 | SlackEnabled: True 11 | ThirdPartyLoggerLevels: 12 | botocore: CRITICAL 13 | 'urllib3.connectionpool': CRITICAL 14 | 15 | TESTING: 16 | ThisIs: A Test 17 | -------------------------------------------------------------------------------- /mkdocs/architecture/ResidentIAMRole.md: -------------------------------------------------------------------------------- 1 | # Resident IAM Role 2 | 3 | For Starfleet worker ships to function, there must be an IAM role (or roles) in all of the accounts that it can assume with the permissions to allow the worker to do the things that it needs to do. 4 | 5 | Starfleet is deployed to 1 AWS account, which is a _**security sensitive account**_ and the IAM roles for the Starfleet worker ships will need `sts:AssumeRole` permissions to perform their workloads. 6 | 7 | The User Guide contains more details on how to set this up, but just know that the account resident IAM Roles exist for Starfleet to work. 8 | -------------------------------------------------------------------------------- /src/starfleet/cli/entrypoint.py: -------------------------------------------------------------------------------- 1 | """The main CLI entrypoint for Starfleet. 2 | 3 | This outlines the main CLI entrypoint objects that are to be used throughout. 4 | 5 | :Module: starfleet.cli.entrypoint 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import click 12 | 13 | from starfleet.cli.components import StarfleetClickGroup 14 | 15 | 16 | @click.group(cls=StarfleetClickGroup) 17 | def cli() -> None: 18 | """Starfleet is a totally awesome whole-infrastructure automation tool.""" 19 | -------------------------------------------------------------------------------- /src/starfleet/account_index/plugins/starfleet_default_index/__init__.py: -------------------------------------------------------------------------------- 1 | """The Starfleet default Account Index Plugin 2 | 3 | This is the module for the Default Account Index plugin that utilizes the index geneated by the AccountIndexGeneratorShip plugin. 4 | 5 | :Module: starfleet.account_index.plugins.starfleet_default_index.ship 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | from starfleet.account_index.plugins.starfleet_default_index.ship import StarfleetDefaultAccountIndex 12 | 13 | ACCOUNT_INDEX_PLUGINS = [StarfleetDefaultAccountIndex] 14 | -------------------------------------------------------------------------------- /ATTRIBUTIONS: -------------------------------------------------------------------------------- 1 | THIS FILE CONTAINS OTHER ATTRIBUTIONS FOR THIS OSS PROJECT: 2 | 3 | 1. Spaceship Icons: 4 | - The spaceship icons are from the *VERY COOL* The Ur-Quan Masters project: https://sc2.sourceforge.net/ 5 | - The Starfleet open source project has no relation to The Ur-Quan Masters project 6 | - The icons were scaled up for size 7 | - The content -- … graphics, … -- are copyright (C) 1992, 1993, 2002 Toys for Bob, Inc. or their respective creators. 8 | The content may be used freely under the terms of the Creative Commons Attribution-NonCommercial-ShareAlike 2.5 license (available at http://creativecommons.org/licenses/by-nc-sa/2.5/). 9 | - The icons are distributed with no warranties of any kind 10 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/plugins/account_index_generator/__init__.py: -------------------------------------------------------------------------------- 1 | """Starfleet's worker for dumping an account inventory to S3. 2 | 3 | This is a worker ship that will periodically dump an inventory of AWS accounts from the organizations API. 4 | 5 | :Module: starfleet.worker_ships.plugins.account_index_generator 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | from starfleet.worker_ships.plugins.account_index_generator.ship import AccountIndexGeneratorShip, account_inventory 12 | 13 | WORKER_SHIP_PLUGINS = [AccountIndexGeneratorShip] 14 | CLICK_CLI_GROUPS = [account_inventory] 15 | -------------------------------------------------------------------------------- /mkdocs/userGuide/PayloadTemplates.md: -------------------------------------------------------------------------------- 1 | # Payload Templates 2 | 3 | Each worker defines their own payload template schema. Please see the navigation on the left to find the respective worker's payload template schema. 4 | 5 | ## Base Template 6 | All worker ships will have the following fields: 7 | 8 | ```yaml 9 | TemplateName: SomeNameForYourTemplate 10 | TemplateDescription: Some description that makes it easy for you to understand what this is for. 11 | ``` 12 | 13 | Each worker ship then defines additional fields as appropriate. 14 | 15 | ## Account and Account Region Payload Reference 16 | The schemas for the Account and Account/Region worker ship payloads [can be found here](../architecture/PayloadTemplates.md#account-worker-templates). -------------------------------------------------------------------------------- /.github/workflows/python.yml: -------------------------------------------------------------------------------- 1 | name: Python package 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | python-version: 14 | - "3.12" 15 | steps: 16 | - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 17 | - name: Set up Python ${{ matrix.python-version }} 18 | uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | - name: Install Tox 22 | run: pip install tox 23 | - name: Run Tox Tests and Lint 24 | run: tox -e py,cloudformation,mkdocs 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Starfleet 2 | 3 | ## Documentation 4 | There is a lot of documentation on our very fancy docs site here: [https://gemini-oss.github.io/starfleet](https://gemini-oss.github.io/starfleet) 5 | 6 | ## What is it? 7 | This is a whole AWS infrastructure automation tool that is able to run Lambda functions with AWS account specific workloads. 8 | It's primary purpose is to enable cloud and infrastructure security teams the ability to run automation for ensuring infrastructure 9 | state across an entire AWS organization in a manner that cannot adequately be achieved with AWS Organizations. 10 | 11 | You can make lambdas to do whatever it is that you want to do with the proper context of the AWS account or region that you want 12 | to operate in. 13 | -------------------------------------------------------------------------------- /mkdocs/blog/posts/fwd-cloudsec.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2023-07-19 3 | authors: 4 | - mikegrima 5 | categories: 6 | - General Announcement 7 | --- 8 | 9 | # Starfleet Presented at fwd:cloudsec 2023 10 | Starfleet was presented at [fwd:cloudsec](https://fwdcloudsec.org/) 2023 in the talk titled [_Rolling out AWS Infrastructure Everywhere with Space Ships_](https://fwdcloudsec.org/speakers.html#rolling-out-aws-infrastructure). 11 | 12 | Watch it [here](https://www.youtube.com/watch?v=5hyLfhTjtmE): 13 | 14 | 15 | -------------------------------------------------------------------------------- /mkdocs/Attributions.md: -------------------------------------------------------------------------------- 1 | # Attributions 2 | 3 | 1. Space ship icons/logos 4 | - The spaceship icons are from the *VERY COOL* The Ur-Quan Masters project: [https://sc2.sourceforge.net/](https://sc2.sourceforge.net/) 5 | - The Starfleet open source project has no relation to The Ur-Quan Masters project 6 | - The icons were scaled up for size 7 | - The content -- … graphics, … -- are copyright (C) 1992, 1993, 2002 Toys for Bob, Inc. or their respective creators. 8 | The content may be used freely under the terms of the Creative Commons Attribution-NonCommercial-ShareAlike 2.5 license (available at [https://creativecommons.org/licenses/by-nc-sa/2.5/](https://creativecommons.org/licenses/by-nc-sa/2.5/)). 9 | - The icons are distributed with no warranties of any kind 10 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/plugins/github_sync/__init__.py: -------------------------------------------------------------------------------- 1 | """Starfleet's worker for syncing a GitHub repository contents to S3 2 | 3 | This is a worker ship that will periodically verify that a GitHub repository's contents are identical to that 4 | of an S3 bucket. It will add, remove, and update any file in S3 that is not in the corresponding GitHub repo. 5 | 6 | This is primarily used for syncing Starfleet templates to S3 for CI/CD, but can be used for any other purpose. 7 | 8 | :Module: starfleet.worker_ships.plugins.github_sync 9 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 10 | :License: See the LICENSE file for details 11 | :Author: Mike Grima 12 | """ 13 | 14 | from starfleet.worker_ships.plugins.github_sync.ship import GitHubSyncWorkerShip, sync_github 15 | 16 | WORKER_SHIP_PLUGINS = [GitHubSyncWorkerShip] 17 | CLICK_CLI_GROUPS = [sync_github] 18 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/plugins/aws_config/__init__.py: -------------------------------------------------------------------------------- 1 | """Starfleet's worker ship for enabling AWS Config recording 2 | 3 | This is a worker ship that will operate on all accounts/regions, and ti will enable AWS Config recorders in all accounts. This will not 4 | enable aggregation, which we are recommending that you leverage the AWS Organizations feature for. But, this worker will allow you to get 5 | AWS Config recording in place such that you are able to make use of the organization aggregation feature. 6 | 7 | :Module: starfleet.worker_ships.plugins.aws_config 8 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 9 | :License: See the LICENSE file for details 10 | :Author: Mike Grima 11 | """ 12 | 13 | from starfleet.worker_ships.plugins.aws_config.ship import AwsConfigWorkerShip, aws_config 14 | 15 | WORKER_SHIP_PLUGINS = [AwsConfigWorkerShip] 16 | CLICK_CLI_GROUPS = [aws_config] 17 | -------------------------------------------------------------------------------- /src/starfleet/utils/niceties.py: -------------------------------------------------------------------------------- 1 | """A general set of niceties that Starfleet can use to do things that are nice. 2 | 3 | This mostly defines some shortcut code utilities that workers can use for a variety of use cases. 4 | 5 | :Module: starfleet.utils.niceties 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | from typing import Set 12 | 13 | import boto3 14 | 15 | 16 | def get_all_regions(service: str = "ec2") -> Set[str]: 17 | """ 18 | This will return all supported AWS regions for the supplied service. By default, this returns the set for EC2. 19 | 20 | This is placed here as a function so that we can easily mock out the values with a static set of values that will persist throughout boto3 updates. 21 | """ 22 | return set(boto3.session.Session().get_available_regions(service)) 23 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | requires = 3 | tox>=4.2 4 | env_list = 5 | py312 6 | lint 7 | no_package = true 8 | 9 | [testenv] 10 | use_develop = true 11 | deps = 12 | . 13 | .[tests] 14 | set_env = 15 | PYTHONPATH = src 16 | commands = 17 | pytest --cov --cov-report term-missing tests -n auto {posargs} 18 | 19 | [testenv:lint] 20 | skip_install = true 21 | commands = 22 | black --check --diff . 23 | flake8 src/starfleet tests/ setup.py 24 | pylint --rcfile=tox.ini src/starfleet tests/ setup.py 25 | 26 | [testenv:reformat] 27 | skip_install = true 28 | commands = 29 | black . 30 | 31 | [testenv:cloudformation] 32 | skip_install = true 33 | commands = 34 | checkov --quiet --directory deploy 35 | cfn-lint deploy/**/*.yml 36 | 37 | [testenv:mkdocs] 38 | skip_install = true 39 | commands = 40 | mkdocs build 41 | 42 | [flake8] 43 | ignore = E501,W503,E203 44 | 45 | [pylint] 46 | disable = C0301,W1203,C0415,W0212,R0903,W0511,R0913,R0801 47 | -------------------------------------------------------------------------------- /tests/test_configuration_files/basic_plugin.yaml: -------------------------------------------------------------------------------- 1 | TestingStarfleetWorkerPlugin: 2 | Enabled: True 3 | TemplatePrefix: TestingStarfleetWorkerPlugin/ 4 | InvocationQueueUrl: https://us-east-2.queue.amazonaws.com/123456789012/WorkerQueue1 5 | InvocationSources: 6 | - S3 7 | - EVENTBRIDGE_TIMED_EVENT 8 | EventBridgeTimedFrequency: HOURLY 9 | 10 | 11 | TestingStarfleetWorkerPluginTwo: 12 | Enabled: True 13 | TemplatePrefix: TestingStarfleetWorkerPluginTwo/ 14 | InvocationQueueUrl: https://us-east-2.queue.amazonaws.com/123456789012/WorkerQueue2 15 | InvocationSources: 16 | - S3 17 | - EVENTBRIDGE_TIMED_EVENT 18 | EventBridgeTimedFrequency: HOURLY 19 | 20 | 21 | TestingStarfleetWorkerPluginThree: 22 | Enabled: True 23 | TemplatePrefix: TestingStarfleetWorkerPluginThree/ 24 | InvocationQueueUrl: https://us-east-2.queue.amazonaws.com/123456789012/WorkerQueue3 25 | InvocationSources: 26 | - S3 27 | - EVENTBRIDGE_TIMED_EVENT 28 | EventBridgeTimedFrequency: DAILY 29 | -------------------------------------------------------------------------------- /mkdocs/userGuide/awsConfig/CLI.md: -------------------------------------------------------------------------------- 1 | # AWS Config Worker CLI 2 | 3 | The AWS Config worker has just 1 CLI command (`sync`), which syncs the AWS Config template to the given AWS Account ID and region in question. 4 | 5 | Below is the CLI options: 6 | 7 | ```bash 8 | Usage: starfleet aws-config sync [OPTIONS] 9 | 10 | This will sync the AWS Config payload in the desired account and region. 11 | 12 | Options: 13 | --payload FILENAME This is the worker payload YAML [required] 14 | --account-id TEXT The AWS account ID to operate in [required] 15 | --region TEXT The AWS region to operate in [required] 16 | --commit Must be supplied for changes to be made 17 | --help Show this message and exit. 18 | ``` 19 | 20 | ## The `aws-config sync` command 21 | Running the sync command requires that you have the required Starfleet AWS credentials exported in the environment. Here is an example of how to run it in commit mode on account ID `111111111111` in `us-east-1`: 22 | 23 | ```bash 24 | starfleet aws-config sync --payload some/path/to/the/payload.yaml --account-id 111111111111 --region us-east-1 --commit 25 | ``` 26 | -------------------------------------------------------------------------------- /src/starfleet/utils/logging.py: -------------------------------------------------------------------------------- 1 | """Starfleet's logger management 2 | 3 | This holds the Starfleet logger, which is to be used throughout the application for all logging and output purposes. 4 | 5 | :Module: starfleet.utils.logging 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import logging 12 | 13 | LOGGER = logging.getLogger("starfleet") 14 | 15 | # Create console handler: 16 | handler = logging.StreamHandler() 17 | 18 | # Create formatter: 19 | formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s - %(pathname)s - %(funcName)s:%(lineno)i") 20 | 21 | # Add formatter to the handler: 22 | handler.setFormatter(formatter) 23 | 24 | # Add the handler to the logger: 25 | LOGGER.addHandler(handler) 26 | 27 | # Documented: https://stackoverflow.com/a/50910770 28 | LOGGER.propagate = False # Prevents the duplicate log entries from appearing in CloudWatch Logs 29 | 30 | # The log level will be set by the configuration. 31 | # The configuration will also update the app's 3rd party logging levels to supress things like urllib and boto since they are noisy. 32 | -------------------------------------------------------------------------------- /mkdocs/developerGuide/primaryComponents/Loggers.md: -------------------------------------------------------------------------------- 1 | # Loggers 2 | 3 | We have a common logger that you should make use of throughout Starfleet. This is configured on startup whenever any Python file imports it. 4 | 5 | To use the logger you need to import it and the interact with it using `LOGGER`: 6 | 7 | ```python 8 | # Import the logger: 9 | from starfleet.utils.logging import LOGGER 10 | 11 | # ... 12 | 13 | # Use the Logger: 14 | LOGGER.info("[🛸] something to log...") 15 | ``` 16 | 17 | The logger is configured to log out everything in a nice format that appears in the Lambda CloudWatch Logs and should appear nicely in any log aggregation system you want to make use of. Things like the log level and 3rd party loggers to ignore is set in the configuration ([details here](../../architecture/Configuration.md#optional-fields)). 18 | 19 | !!! tip 20 | One thing you will commonly see throughout the Starfleet codebase are log entries with emojis wrapped in brackets. We just ❤️ Emojis here (and believe it or not, it makes it easier to locate entires in a log system 🤣) 21 | 22 | *Protip:* make a bookmark for [https://emojipedia.org/](https://emojipedia.org/) 23 | 24 | If you want to see the raw code for the logger take a look at `starfleet.utils.logging`. 25 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/plugins/iam/__init__.py: -------------------------------------------------------------------------------- 1 | """Starfleet workers for IAM related things 2 | 3 | These are worker ships for IAM related things, like roles. 4 | 5 | :Module: starfleet.worker_ships.plugins.iam 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import click 12 | 13 | from starfleet.worker_ships.plugins.iam.role_ship import IamRoleWorkerShip, role 14 | 15 | 16 | # Roll up the IAM CLI commands under one group: 17 | @click.group() 18 | def iam() -> None: 19 | """ 20 | This is the parent command group for all Starfleet worker commands. Each worker implements their own subcommands for this. 21 | 22 | All IAM workers leverage the 3rd party iambic library. Please see iambic.org for details on how to write IAM templates. Also, check out the Starfleet 23 | documentation on this worker as Starfleet has some implementation specific details. 24 | 25 | Note: The account index is utilized for these commands and as such, AWS credentials may be required to run them. 26 | """ 27 | 28 | 29 | # Add in role commands: 30 | iam.add_command(role) # noqa 31 | 32 | 33 | WORKER_SHIP_PLUGINS = [IamRoleWorkerShip] 34 | CLICK_CLI_GROUPS = [iam] 35 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "monthly" 12 | groups: 13 | # This is the name of your group, it will be used in PR titles and branch names 14 | primary-deps: 15 | patterns: 16 | - "pyjwt" 17 | - "cryptography" 18 | - "click" 19 | - "requests" 20 | - "retry" 21 | - "slack-sdk" 22 | - "pyyaml" 23 | - "marshmallow" 24 | - "boto*" 25 | - "cloudaux-lite" 26 | iambic-dependencies: 27 | patterns: 28 | - "iambic-core" 29 | primary-dev-dependencies: 30 | patterns: 31 | - "tox" 32 | - "mkdocs*" 33 | - "pytest*" 34 | - "moto" 35 | - "cfn-lint" 36 | - "checkov" 37 | - "black" 38 | - "pylint" 39 | - "flake8" 40 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/iam/bad_payloads/bad_role_iambic.yaml: -------------------------------------------------------------------------------- 1 | TemplateName: BadIamRoleIambicValidationTest 2 | TemplateDescription: This is a bad IAM role template with a problem that will fail iambic validation 3 | IncludeAccounts: 4 | AllAccounts: True 5 | IambicVariables: 6 | - Key: Some-Key 7 | Value: Some-Value 8 | IambicRoleTemplate: 9 | properties: 10 | description: 'Starfleet iambic test role with variable {{ var.Some-Key }}' 11 | assume_role_policy_document: 12 | statement: 13 | - action: 14 | - sts:AssumeRole 15 | effect: Allow 16 | principal: 17 | service: lambda.amazonaws.com 18 | version: '2012-10-17' 19 | managed_policies: 20 | - policy_arn: arn:aws:iam::aws:policy/ReadOnlyAccess 21 | inline_policies: 22 | - policy_name: 'SomePolicyIn-{{ var.account_name }}' 23 | StarfleetIncludeAccounts: 24 | ByOrgUnits: 25 | - SomeNestedOU 26 | StarfleetExcludeAccounts: 27 | ByNames: 28 | - Account 10 29 | statement: 30 | - effect: Deny 31 | action: 32 | - 33 | - 34 | - 35 | resource: '*' 36 | role_name: StarfleetIambicTesting 37 | tags: 38 | - key: owner 39 | value: pewpewpew 40 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "starfleet" 7 | authors = [ 8 | {name = "Gemini", email = "careers@gemini.com"}, 9 | {name = "Mike Grima", email = "michael.grima@gemini.com"}, 10 | ] 11 | version = "1.0.2" 12 | dynamic = ["dependencies"] 13 | 14 | [tool.setuptools] 15 | package-dir = {"" = "src"} 16 | 17 | [tool.setuptools.packages.find] 18 | where = ["src"] 19 | 20 | [tool.setuptools.package-data] 21 | "starfleet.configuration_files" = ["*.yaml"] 22 | 23 | [tool.setuptools.dynamic] 24 | dependencies = {file = ["src/requirements.txt"]} 25 | 26 | [project.scripts] 27 | starfleet = "starfleet.cli.entrypoint:cli" 28 | 29 | [project.optional-dependencies] 30 | tests = [ 31 | "pytest==8.1.1", 32 | "pytest-cov==5.0.0", 33 | "pytest-xdist==3.5.0", 34 | "black==24.3.0", 35 | "flake8==7.0.0", 36 | "pylint==3.1.0", 37 | "tox==4.14.2", 38 | "moto==5.0.4", 39 | "mkdocs==1.5.3", 40 | "mkdocstrings[python]==0.24.1", 41 | "mkdocs-gen-files==0.5.0", 42 | "mkdocs-literate-nav==0.6.1", 43 | "mkdocs-material==9.5.16", 44 | "cfn-lint>=0.77", 45 | "checkov>=2.3", 46 | "setuptools==69.2.0" 47 | ] 48 | 49 | [tool.pytest.ini_options] 50 | log_cli = true 51 | 52 | [tool.black] 53 | line-length = 160 54 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/iam/test_role_payload.yaml: -------------------------------------------------------------------------------- 1 | TemplateName: TestIamRoleTemplate 2 | TemplateDescription: This is a test IAM Role template 3 | IncludeAccounts: 4 | AllAccounts: True 5 | IambicVariables: 6 | - Key: some_key 7 | Value: some_value 8 | - Key: some_other_key 9 | Value: some_other_value 10 | IambicRoleTemplate: 11 | properties: 12 | description: 'Starfleet iambic test role with variable {{var.some_key}}' 13 | assume_role_policy_document: 14 | statement: 15 | - action: sts:AssumeRole 16 | effect: Allow 17 | principal: 18 | service: ec2.amazonaws.com 19 | version: '2012-10-17' 20 | managed_policies: 21 | - policy_arn: arn:aws:iam::aws:policy/ReadOnlyAccess 22 | inline_policies: 23 | - policy_name: 'SomePolicyIn-{{var.account_name}}' 24 | StarfleetIncludeAccounts: 25 | ByOrgUnits: 26 | - SomeNestedOU 27 | StarfleetExcludeAccounts: 28 | ByNames: 29 | - Account 10 30 | statement: 31 | - effect: Deny 32 | action: s3:* 33 | resource: '*' 34 | version: '2012-10-17' 35 | role_name: StarfleetIambicTesting 36 | tags: 37 | - key: owner 38 | value: pewpewpew 39 | - key: some_other_key 40 | value: '{{var.some_other_key}}' 41 | -------------------------------------------------------------------------------- /tests/account_index/conftest.py: -------------------------------------------------------------------------------- 1 | """PyTest fixtures for Starfleet's Account Indexer Tests 2 | 3 | This defines the PyTest fixtures for the Account Indexer tests 4 | 5 | :Module: starfleet.tests.account_index.conftest 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | # pylint: disable=unused-argument 12 | from typing import Generator, Any, Dict 13 | 14 | import pytest 15 | 16 | from starfleet.account_index.loader import AccountIndexInstance 17 | 18 | 19 | @pytest.fixture 20 | def test_index(test_configuration: Dict[str, Any]) -> Generator[AccountIndexInstance, None, None]: 21 | """This returns the StarfleetAccountIndexLoader with a TestingAccountIndexPlugin mocked out for it. This mocks out for the entire app.""" 22 | from starfleet.account_index.loader import ACCOUNT_INDEX, StarfleetAccountIndexLoader 23 | import tests.account_index.testing_plugins 24 | 25 | account_indexer = StarfleetAccountIndexLoader() 26 | account_indexer._index_ship_path = tests.account_index.testing_plugins.__path__ 27 | account_indexer._index_ship_prefix = tests.account_index.testing_plugins.__name__ + "." 28 | 29 | ACCOUNT_INDEX._index = account_indexer.index 30 | yield ACCOUNT_INDEX.index 31 | 32 | ACCOUNT_INDEX.reset() 33 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/iam/bad_payloads/bad_role_marshmallow.yaml: -------------------------------------------------------------------------------- 1 | TemplateName: BadIamRoleMarshmallowValidationTest 2 | TemplateDescription: This is a bad IAM role template with a problem that will fail marshmallow validation 3 | IncludeAccounts: 4 | AllAccounts: True 5 | IambicVariables: 6 | - Key: Some-Key 7 | Value: Some-Value 8 | IambicRoleTemplate: 9 | properties: 10 | description: 'Starfleet iambic test role with variable {{ var.Some-Key }}' 11 | included_accounts: # Can't have this field -- need to use the StarfleetIncludeAccounts field. 12 | - '*' 13 | assume_role_policy_document: 14 | statement: 15 | - action: 16 | - sts:AssumeRole 17 | effect: Allow 18 | principal: 19 | service: lambda.amazonaws.com 20 | version: '2012-10-17' 21 | managed_policies: 22 | - policy_arn: arn:aws:iam::aws:policy/ReadOnlyAccess 23 | inline_policies: 24 | - policy_name: 'SomePolicyIn-{{ var.account_name }}' 25 | StarfleetIncludeAccounts: 26 | ByOrgUnits: 27 | - SomeNestedOU 28 | StarfleetExcludeAccounts: 29 | ByNames: 30 | - Account 10 31 | statement: 32 | - effect: Deny 33 | action: s3:* 34 | resource: '*' 35 | role_name: StarfleetIambicTesting 36 | tags: 37 | - key: owner 38 | value: pewpewpew 39 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Sample Starfleet Dockerfile for building ECR Lambda functions 2 | FROM public.ecr.aws/lambda/python:3.12 3 | 4 | ENV LAMBDA_TASK_ROOT=/var/runtime 5 | 6 | # Copy Starfleet over: 7 | COPY ./ ${LAMBDA_TASK_ROOT}/starfleet 8 | 9 | # Perform cleanup of things the Docker doens't need: 10 | RUN rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/tests && \ 11 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/.tox && \ 12 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/.pytest_cache && \ 13 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/deploy && \ 14 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/site && \ 15 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/.kubelr && \ 16 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/build && \ 17 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/.coverage && \ 18 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/*.ini && \ 19 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/*.yaml && \ 20 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/*.md && \ 21 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/sample_samconfig.toml && \ 22 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/Dockerfile && \ 23 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/venv && \ 24 | rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/env 25 | 26 | # Install the specified packages: 27 | RUN cd ${LAMBDA_TASK_ROOT}/starfleet && \ 28 | pip install . && \ 29 | rm -Rf ${LAMBDA_TASK_ROOT}/boto* # Remove the lambda provided boto since it interferes with starfleet 30 | 31 | # The CMD is passed in as ImageConfig in the SAM template 32 | -------------------------------------------------------------------------------- /mkdocs/developerGuide/GeneratingDocs.md: -------------------------------------------------------------------------------- 1 | # Generating the Documentation 2 | 3 | This page outlines how to generate the lovely docs you are currently reading! 4 | 5 | The docs are generated via [`mkdocs`](https://www.mkdocs.org) with [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/). All of the docs are markdown files that reside in the `mkdocs/` directory, and the docs site is configured via the `mkdocs.yml` file. 6 | 7 | ## Dependencies 8 | All of the dependencies to build the docs are installed as part of the test dependencies (defined in `pyproject.toml`). This is obtained by running: 9 | 10 | ```bash 11 | source venv/bin/activate # Make sure you are in your venv 12 | pip install -e ."[tests]" 13 | ``` 14 | 15 | ## Running mkdocs 16 | You can run `mkdocs` locally by running: 17 | 18 | ```bash 19 | source venv/bin/activate # Make sure you are in your venv 20 | mkdocs serve 21 | ``` 22 | 23 | ... and then opening your web browser to `http://localhost:8000`. Errors and warnings will appear in the console output. Please make sure there are none before submitting documentation updates. 24 | 25 | ## Building the docs 26 | The docs are built by running: 27 | 28 | ```bash 29 | source venv/bin/activate # Make sure you are in your venv 30 | mkdocs build -d docs/ 31 | ``` 32 | 33 | Doing this will generate the docs to the `site/` directory. You need to rename this directory to `docs/`. This directory is _not_ included in the _main_ branch of the repository. Instead, we have another branch called `gh-pages` where this is committed to. This is what GitHub uses to host the site you are reading right now. 34 | -------------------------------------------------------------------------------- /mkdocs/userGuide/AccountIndexGenerator.md: -------------------------------------------------------------------------------- 1 | # Account Index Generator User Guide 2 | This is the main user guide for the Account Index Generator worker plugin. Most of the user guide details for the Account Index Generator has been covered in the Architecture and Installation docs (and also in the developer guide). Thus, this page will not be very large. 3 | 4 | ## How to use it? 5 | !!! info "Read this first!" 6 | Make sure that for the first time you are using Starfleet, that you have the configuration properly configured to point to the correct S3 bucket to generate the index JSON to, and that you also have the proper IAM roles present. Remember, you need an IAM Role for the Starfleet workers to assume into in all AWS accounts in your organization, and you also need to make sure that the organization root has this account as well. 7 | 8 | If you choose to leverage CloudFormation StackSets to deploy these roles, remember that StackSets doesn't operate in the organization root account. So you will need to either make a separate CloudFormation stack in the organization root account or just manually create the IAM roles in question. 9 | 10 | In all cases, the configuration needs to be set properly. See the [Installation Guide](../installation/IAM.md) for details. 11 | 12 | The primary things to keep in mind for using Account Index Generator is that: 13 | 14 | 1. All of the components are actually deployed 15 | 1. The IAM Roles are configured as depicted in the Installation Guide 16 | 1. The payload template is configured to properly deploy to the S3 bucket 17 | 1. The Starbase is configured properly 18 | 19 | Of course, please consult the logs for details on what's going on. 20 | 21 | -------------------------------------------------------------------------------- /src/starfleet/startup.py: -------------------------------------------------------------------------------- 1 | """The main module for Starfleet's startup. 2 | 3 | This contains the basic code for startup in Starfleet. All Lambda and CLI invocations will need to execute this. 4 | 5 | :Module: starfleet.startup 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | from starfleet.utils.logging import LOGGER # noqa pylint: disable=W0611 12 | from starfleet.utils.configuration import STARFLEET_CONFIGURATION 13 | from starfleet.account_index.loader import ACCOUNT_INDEX 14 | from starfleet.worker_ships.loader import STARFLEET_WORKER_SHIPS 15 | 16 | 17 | def base_start_up() -> None: 18 | """This is a function that will execute all startup related tasks that needs to be performed for Starfleet to function. 19 | 20 | The start-up order is as follows: 21 | 1. Load the base configuration 22 | 2. Set up the logger (TODO: add metrics plugins) 23 | """ 24 | # Step 1: Load the base configuration, which also configures the logger for the app: 25 | STARFLEET_CONFIGURATION.config # noqa pylint: disable=pointless-statement 26 | 27 | 28 | def starbase_start_up() -> None: 29 | """This is a function that will execute all startup related tasks that needs to be performed for Starfleet's starbase to function. 30 | 31 | The start-up order is as follows: 32 | 1. Load the base startup 33 | 2. Set up the account index 34 | 3. Set up the worker ships 35 | """ 36 | base_start_up() 37 | 38 | # Account Index: 39 | ACCOUNT_INDEX.index # noqa pylint: disable=pointless-statement 40 | 41 | # Worker Ships: 42 | STARFLEET_WORKER_SHIPS.get_worker_ships() # noqa pylint: disable=pointless-statement 43 | -------------------------------------------------------------------------------- /sample_samconfig.toml: -------------------------------------------------------------------------------- 1 | version = 0.1 2 | [TEST] 3 | [TEST.deploy] 4 | [TEST.deploy.parameters] 5 | stack_name = "starfleet" 6 | s3_bucket = "REPLACE-ME" # The SAM CLI will generate this with a "guided" deploy option -- or you can just make this yourself. 7 | s3_prefix = "starfleet" 8 | region = "REPLACEME" 9 | confirm_changeset = true 10 | capabilities = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"] # Important -- you need these capabilities defined since this creates IAM roles 11 | parameter_overrides = "EnvironmentName=\"TEST\"" 12 | image_repository = "REPLACE ME - IF YOU ARE NOT USING ECR DELETE THIS LINE" 13 | 14 | [TEST.validate.parameters] 15 | region = "REPLACEME" 16 | lint = true 17 | template_file = "test_sam_template.yaml" # Feel free to replace with your own filename 18 | 19 | [TEST.build.parameters] 20 | use_container = true 21 | template_file = "test_sam_template.yaml" # Feel free to replace with your own filename 22 | 23 | 24 | [PROD] 25 | [PROD.deploy] 26 | [PROD.deploy.parameters] 27 | stack_name = "starfleet" 28 | s3_bucket = "REPLACE-ME" # The SAM CLI will generate this with a "guided" deploy option -- or you can just make this yourself. 29 | s3_prefix = "starfleet" 30 | region = "REPLACEME" 31 | confirm_changeset = true 32 | capabilities = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"] # Important -- you need these capabilities defined since this creates IAM roles 33 | parameter_overrides = "EnvironmentName=\"PROD\"" 34 | image_repository = "REPLACE ME - IF YOU ARE NOT USING ECR DELETE THIS LINE" 35 | 36 | [PROD.validate.parameters] 37 | region = "REPLACEME" 38 | lint = true 39 | template_file = "prod_sam_template.yaml" # Feel free to replace with your own filename 40 | 41 | [PROD.build.parameters] 42 | use_container = true 43 | template_file = "prod_sam_template.yaml" # Feel free to replace with your own filename 44 | -------------------------------------------------------------------------------- /src/starfleet/starbase/entrypoints.py: -------------------------------------------------------------------------------- 1 | """Starbase Lambda Entrypoints 2 | 3 | All of the Lambda entrypoints for the Starbase are here. 4 | 5 | :Module: starfleet.starbase.entrypoints 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import json 12 | from typing import Any, Dict 13 | 14 | from starfleet.startup import starbase_start_up 15 | from starfleet.starbase.main import fan_out_payload, process_eventbridge_timed_event 16 | from starfleet.utils.logging import LOGGER 17 | 18 | 19 | def eventbridge_timed_lambda_handler(event: Dict[str, Any], context: object) -> None: # noqa pylint: disable=W0613 20 | """This is the Lambda entrypoint for the EventBridge timed events.""" 21 | starbase_start_up() 22 | 23 | LOGGER.info("[🎬] Starting Starbase for EventBridge timed event...") 24 | process_eventbridge_timed_event(event) 25 | LOGGER.info("[🏁] Completed Starbase EventBridge timed event.") 26 | 27 | 28 | def fanout_payload_lambda_handler(event: Dict[str, Any], context: object) -> None: # noqa pylint: disable=W0613 29 | """This is the Lambda entrypoint that will fan out the workload to all worker ships for the given template.""" 30 | starbase_start_up() 31 | 32 | LOGGER.info("[🎬] Starting Starbase Worker Ship fanout...") 33 | 34 | # This should not!! be a list, but if it is, just handle it anyway: 35 | if len(event["Records"]) > 1: 36 | LOGGER.error("[🚨] Received more than 1 event for fan out! This should only receive 1, but handling it anyway...") 37 | 38 | for record in event["Records"]: 39 | payload = json.loads(record["body"]) 40 | fan_out_payload(payload) 41 | 42 | LOGGER.info("[🏁] Completed Starbase Worker Ship fanout.") 43 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/conftest.py: -------------------------------------------------------------------------------- 1 | """PyTest fixtures for Starfleet's Worker Ships 2 | 3 | This defines the PyTest fixtures that can be used by all worker ship tests. 4 | 5 | :Module: starfleet.tests.worker_ship_utils.conftest 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | # pylint: disable=unused-argument 12 | 13 | from typing import Any, Dict, Generator 14 | from unittest import mock 15 | from unittest.mock import MagicMock 16 | 17 | import pytest 18 | import yaml 19 | 20 | SAMPLE_GOOD_CONFIG = """ 21 | Enabled: False 22 | TemplatePrefix: somePrefix/ 23 | InvocationQueueUrl: https://sqs.amazonaws.com/SomeQueueUrl 24 | InvocationSources: 25 | - S3 26 | - EVENTBRIDGE_TIMED_EVENT 27 | EventBridgeTimedFrequency: HOURLY 28 | """ 29 | 30 | 31 | SAMPLE_BASE_PAYLOAD_TEMPLATE = """ 32 | TemplateName: SampleBasePayloadTemplate 33 | TemplateDescription: This is a template for unit test purposes only. 34 | """ 35 | 36 | 37 | @pytest.fixture 38 | def sample_good_config() -> Dict[str, Any]: 39 | """This returns the SAMPLE_GOOD_CONFIG as a dictionary for use in testing schemas.""" 40 | return yaml.safe_load(SAMPLE_GOOD_CONFIG) 41 | 42 | 43 | @pytest.fixture 44 | def sample_payload_template() -> Dict[str, Any]: 45 | """This returns the SAMPLE_BASE_PAYLOAD_TEMPLATE as a dictionary for use in testing schemas.""" 46 | return yaml.safe_load(SAMPLE_BASE_PAYLOAD_TEMPLATE) 47 | 48 | 49 | @pytest.fixture 50 | def mock_loader_logger() -> Generator[MagicMock, None, None]: 51 | """This will mock out the logger that is used during the worker ship loading and return a MagicMock for tests to verify that log entries are being made.""" 52 | with mock.patch("starfleet.worker_ships.loader.LOGGER") as mock_logger: 53 | yield mock_logger 54 | -------------------------------------------------------------------------------- /mkdocs/userGuide/IAM/Roles/CLI.md: -------------------------------------------------------------------------------- 1 | # IAM Role Worker CLI 2 | 3 | The IAM Role worker CLI resides under the `iam role` sub-command. Below are the CLI options 4 | 5 | ```bash 6 | starfleet iam role 7 | Usage: starfleet iam role [OPTIONS] COMMAND [ARGS]... 8 | 9 | This is the worker ship for processing a Starfleet-wrapped iambic.org IAM 10 | role template. 11 | 12 | Note: The account index is utilized for these commands and as such, AWS 13 | credentials may be required to run them. 14 | 15 | Options: 16 | --help Show this message and exit. 17 | 18 | Commands: 19 | sync This will invoke iambic to sync out the IAM role. 20 | validate-iambic This will validate the supplied Starfleet-wrapped... 21 | ``` 22 | 23 | ## The `iam role validate-iambic` command 24 | This command is used to ensure that template itself is mostly well formed. This will not perform any account resolution logic and is mostly used to confirm that the template will make its way to IAMbic and that IAMbic is satisfied with the template. 25 | 26 | This command does _not_ require AWS credentials and does not take in any arguments other than the path to the file to validate. The `commit` flag has no effect on this command. Example: 27 | 28 | ```bash 29 | starfleet iam role --payload some/path/to/the/payload.yaml 30 | ``` 31 | 32 | The output will inform you if there are any problems or not and what fixes should be performed if there are any issues. 33 | 34 | ## The `iam role sync` command 35 | This is analogous to all the other `sync` commands for the other workers. This command does require AWS credentials and will perform both the validation for a given AWS account and if the `commit` flag is supplied, it will perform any changes required. 36 | 37 | Here is an example of how to run it in commit mode on account ID `111111111111`: 38 | 39 | ```bash 40 | starfleet iam role sync --payload some/path/to/the/payload.yaml --account-id 111111111111 --commit 41 | ``` 42 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/plugins/iam/iambic_imports.py: -------------------------------------------------------------------------------- 1 | """File to handle the IAMbic imports. This is done to avoid logger issues. 2 | 3 | :Module: starfleet.worker_ships.plugins.iam.iambic_imports 4 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 5 | :License: See the LICENSE file for details 6 | :Author: Mike Grima 7 | """ 8 | 9 | # pylint: disable=unused-import,wrong-import-position,wrong-import-order 10 | from unittest import mock 11 | 12 | from starfleet.utils.logging import LOGGER 13 | 14 | 15 | def _format_message(message: str, **kwargs) -> str: 16 | """Takes in the logging message and the kwargs and formats them so the default Python logger will be happy.""" 17 | return f"{message} - logger_kwargs: {kwargs}" 18 | 19 | 20 | class StarfleetIAMbicLoggerOverrider: 21 | """Class that addresses IAMbic's custom logging functionality.""" 22 | 23 | def debug(self, text: str, **kwargs) -> None: 24 | """Mocks out the IAMbic logger debug log.""" 25 | LOGGER.debug(_format_message(text, **kwargs)) 26 | 27 | def info(self, text: str, **kwargs) -> None: 28 | """Mocks out the IAMbic logger info log.""" 29 | 30 | LOGGER.info(_format_message(text, **kwargs)) 31 | 32 | def error(self, text: str, **kwargs) -> None: 33 | """Mocks out the IAMbic logger error log.""" 34 | 35 | LOGGER.error(_format_message(text, **kwargs)) 36 | 37 | 38 | iambic_logger = StarfleetIAMbicLoggerOverrider() 39 | 40 | # Mock out the iambic logger, as it's bossy and overrides Starfleet's: 41 | mock.patch("iambic.core.logger.log", iambic_logger).start() 42 | 43 | # Now continue to import the rest: 44 | from iambic.core.context import ctx as iambic_ctx # noqa: E402,F401 45 | from iambic.core.models import Variable # noqa: E402,F401 46 | from iambic.plugins.v0_1_0.aws.iam.role.models import AwsIamRoleTemplate # noqa: E402,F401 47 | from iambic.plugins.v0_1_0.aws.models import AWSAccount # noqa: E402,F401 48 | -------------------------------------------------------------------------------- /mkdocs/architecture/AccountIndex.md: -------------------------------------------------------------------------------- 1 | # Account Index 2 | Starfleet has the concept of an account index. This is an inventory of all AWS accounts that Starfleet can operate over. For simplicity and clarity of documentation, we are going to make the assumption that you are a user of AWS Organizations and have exactly one AWS Organization with many accounts under it. 3 | 4 | The account index keeps a list of accounts that at a minimum need to keep track of: 5 | 6 | 1. AWS account IDs 7 | 1. Enabled regions for accounts 8 | 1. AWS account names 9 | 1. AWS account tags 10 | 1. Organization Units (if applicable) 11 | 1. Organization Roots (if applicable) 12 | 13 | The payload templates will specify the account identifiers listed above for `ACCOUNT` or `ACCOUNT-REGION` payload templates (via the `IncludeAccounts` or `ExcludeAccounts` directives [as documented here](PayloadTemplates.md#account-worker-templates)). The account indexer allows Starfleet to figure out which accounts (or accounts/regions) need to be tasked with a given payload. 14 | 15 | The Account Indexer is also a "ship" in Starfleet (i.e. it's a plugin). The name of the plugin must be present within the `STARFLEET` configuration under the field `AccountIndex`. Example: 16 | ```yaml 17 | AccountIndex: StarfleetDefaultAccountIndex 18 | ``` 19 | 20 | !!! note 21 | Starfleet can support any number of accounts and organizations. As long as your account indexer can index it, it can support it! 22 | 23 | ## Default Account Indexer 24 | Starfleet includes with an account index generator worker ship (`AccountIndexGeneratorShip`) that lists all the accounts in AWS Organizations, fetches their enabled regions, tags, and parent OUs, and then saves this as a JSON file to an S3 bucket. 25 | 26 | In addition to the worker ship for generating the index, Starfleet also ships with an account index plugin (`StarfleetDefaultAccountIndex`) that leverages the saved JSON from the `AccountIndexGeneratorShip` to provide Starfleet with the account index capabilities. 27 | 28 | This is described in much more detail in the User Guide. 29 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/aws_config/test_payload.yaml: -------------------------------------------------------------------------------- 1 | TemplateName: AWSConfigEnablement 2 | TemplateDescription: Enabled AWS Config everywhere 3 | IncludeAccounts: 4 | AllAccounts: True 5 | OperateInOrgRoot: True 6 | IncludeRegions: 7 | - ALL 8 | AccountOverrideConfigurations: 9 | - 10 | IncludeAccounts: 11 | ByIds: 12 | - "000000000001" 13 | IncludeRegions: 14 | - us-west-1 15 | DeliveryChannelDetails: 16 | BucketName: bucket-000000000001 17 | BucketKeyPrefix: some/prefix/ 18 | S3DeliveryFrequency: TwentyFour_Hours 19 | S3KmsKeyArn: arn:aws:kms:us-west-1:000000000001:key/1234-1445-1919232 20 | SnsTopicArn: arn:aws:sns:us-west-1:000000000001:topic/sometopic 21 | PreferredName: us-west-1-000000000001 22 | RecorderConfiguration: 23 | PreferredName: us-west-1-000000000001 24 | ConfigRoleName: MyConfigRole 25 | RecordingEnabled: True 26 | RecordingGroup: 27 | RecordEverything: 28 | RecordGlobalsInTheseRegions: 29 | - us-west-1 30 | RetentionPeriodInDays: 30 31 | - 32 | IncludeAccounts: 33 | ByIds: 34 | - "000000000002" 35 | IncludeRegions: 36 | - us-west-1 37 | DeliveryChannelDetails: 38 | BucketName: bucket-000000000002 39 | S3DeliveryFrequency: TwentyFour_Hours 40 | RecorderConfiguration: 41 | ConfigRoleName: MyConfigRole 42 | RecordingEnabled: True 43 | RecordingGroup: 44 | RecordSpecificResources: 45 | - AWS::S3::Bucket 46 | - AWS::EC2::SecurityGroup 47 | RetentionPeriodInDays: 2557 48 | DefaultConfiguration: 49 | DeliveryChannelDetails: 50 | BucketName: all-bucket 51 | S3DeliveryFrequency: TwentyFour_Hours 52 | RecorderConfiguration: 53 | ConfigRoleName: MyConfigRole 54 | RecordingEnabled: True 55 | RecordingGroup: 56 | RecordEverything: 57 | RecordGlobalsInTheseRegions: 58 | - us-east-1 59 | RetentionPeriodInDays: 2557 60 | -------------------------------------------------------------------------------- /mkdocs/userGuide/Starbase.md: -------------------------------------------------------------------------------- 1 | # Starbase User Guide 2 | This is the main user guide for the Starbase component. Most of the user guide details for the Starbase has been covered in the Architecture and Installation docs (and also in the developer guide). 3 | 4 | ## How to use it? 5 | The Starbase is the heart of Starfleet. The only way to use it is to make sure that it's configured properly. The Starbase must also have a proper account inventory prepared, which we discussed in the [installation guide](../installation/MakeAccountIndex.md). 6 | 7 | The primary things to keep in mind for using Starfleet is that: 8 | 9 | 1. All of the components are actually deployed 10 | 1. The account index needs to be available - without this the Starbase has no way of knowing which AWS accounts and/or regions to task the workers for 11 | 1. The template S3 bucket needs to contain the payloads for the workers _AND_ those payloads need to be properly conformant with the corresponding worker payload schemas 12 | 1. The template file names end in `.yaml` 13 | 1. The Starbase IAM role need to have access to the template bucket and also the corresponding worker SQS queues 14 | 1. The configuration for the Starbase (the `STARFLEET` section) and the workers have all the correct details on which SQS queues to use, the template bucket, and the template prefixes within the bucket 15 | 1. The EventBridge timed events are _enabled_ for the given timing task 16 | 1. The Lambda function permissions and the SQS queue permissions are configured to allow Lambda to pull events off of them (this should not be a problem if you use the included AWS SAM templates) 17 | 1. For the Starbase specifically, you will want to ensure that it will operate with an SQS batch size of 1 for fanouts (this is also configured in the SAM template) 18 | 19 | Other than that, the Starbase is mostly not something that you need to worry about, but is something that you should review the logs of if you aren't seeing a worker get tasked. The Starbase is split across 2 Lambda functions: 20 | 21 | 1. The EventBridge Timed Event responder 22 | 2. The worker Fan Out - this is also invoked directly by the template S3 bucket on template object uploads 23 | 24 | If you aren't seeing a worker get invoked, definitely review the logs of each one to see what's going on. 25 | -------------------------------------------------------------------------------- /mkdocs/installation/WrapUp.md: -------------------------------------------------------------------------------- 1 | # Wrapping It Up 2 | 3 | 🤞Hopefully🤞 everything was successful in deploying and getting the account index working. If there were error messages you will want to review them and double-check that everything is set up properly. 4 | 5 | ## Debugging 6 | Debugging is best accomplished by reviewing the logs. You can review the logs in the AWS console for your Starfleet account by going to CloudWatch in the region Starfleet is deployed in, and reviewing the Log Groups for the Lambdas. AWS SAM will generally name these according to the entry in the SAM template YAML. 7 | 8 | You will want to verify that the `/aws/lambda/starfleet-StarbaseFanoutFunction-...`, `/aws/lambda/starfleet-StarbaseEventBridgeFunction-...` (this should be made when the EventBridge events are fired off), and the `/aws/lambda/starfleet-AccountIndexGenerator-...` exist. 9 | 10 | The logs are quite verbose and should contain a lot of details about errors that are caused. In general, you will want to check the EventBridge function's logs to see if there are issues with the Starbase EventBridge scheduling, the Starbase fan out function's logs for issues with fanning out workers, and the individual worker logs to see if there are issues with them. 11 | 12 | !!! tip 13 | By default, CloudWatch logs have no expiration. To save money, set a retention policy on the log groups to have logs automatically deleted after a certain period of time. You need to do this manually... or write a Starfleet worker to do this everywhere... 😏 14 | 15 | ## Deploy to Production 16 | Deploying to production is basically the same exact steps as test, but you will want to replace the test values with production ones. Don't forget to re-run the SAM guided deploy in Prod and correct the SAM configuration so that it's identical to test with production values. 17 | 18 | ## Deployment Considerations 19 | Now that you have Starfleet up and running, you may be thinking about how to manage the templates and all your environment specific configurations (and even custom workers). 20 | 21 | The [Developer Guide](../developerGuide/Overview.md#packaging-deployment-considerations) has some suggestions on how to set up a proper deployment package. The TL;DR is you want a script that would download the upstream Starfleet, and merge in your configuration YAMLs, run SAM, and then perform the deployment. 22 | -------------------------------------------------------------------------------- /mkdocs/userGuide/GitHubSync/Overview.md: -------------------------------------------------------------------------------- 1 | # GitHub Repository Sync Worker User Guide 2 | This is the main user guide for the GitHub Repository Sync Worker ship. This page documents all that there is to know about this worker ship. 3 | 4 | ## What does it do? 5 | This worker ship periodically syncs a GitHub repository with an S3 bucket. The primary purpose of this worker is to enable CI/CD capabilities in Starfleet. CI/CD happens when a repository on GitHub with the Starfleet payload template YAML files are synced with Starfleet's template S3 bucket. Whenever objects are placed into the bucket, a notification is sent out to the Starbase FanOut function, which will then fetch the updated template YAML and task the corresponding worker ship for processing. 6 | 7 | However, this can be used for many GitHub -> S3 use cases. 8 | 9 | ## How it works 10 | The GitHub Sync worker is a `SINGLE_INVOCATION` worker, no AWS account or region context is required. This works by: 11 | 12 | 1. Authenticating as a GitHub App to your GitHub organization 13 | 1. Downloading the .zip of the specified repository's branch 14 | 1. Optionally - extract the .zip file 15 | 1. Compare the downloaded data (either the raw .zip or the extracted files) with the data in S3. If the files are different, then it will upload them. 16 | 1. It will optionally delete files on the bucket that are not present in the repository 17 | 18 | The worker will determine that a file needs to be uploaded if the file is missing in S3 or if the file in S3 has a checksum (S3 ETag) that is not the same as the file in the repo. 19 | 20 | ## Recommended use cases 21 | This worker is optimal for the use case of syncing small text files to S3. I.e. the case of syncing Starfleet payload YAML templates. It is *not* recommended for syncing very large files or syncing a very large number of files. Keep in mind that this runs in a Lambda function so anything that can be accomplished by Lambda is what this is optimal for. 22 | 23 | This worker could be a good use case for syncing static files to an S3 bucket to source a CloudFront distribution. 24 | 25 | We also recommend that you run this worker with an EventBridge schedule set to `FIVE_MINUTES`. 26 | 27 | ## Alerting 28 | The GitHub Sync worker supports alerts to Slack. It will alert on any errors that are encountered during execution. It will also emit `IMPORTANT` notices if it makes any changes to S3. 29 | -------------------------------------------------------------------------------- /mkdocs/installation/PrepareConfiguration.md: -------------------------------------------------------------------------------- 1 | # Prepare Configuration For Installation 2 | 3 | Starfleet needs a proper configuration to function properly. Let's revisit the main configuration for Starfleet. As mentioned in the [Architecture](../architecture/Configuration.md) section, all the configuration files are YAML files that reside in `src/starfleet/configuration_files/`. 4 | 5 | We include a sample [`configuration.yaml`](https://github.com/gemini-oss/starfleet/blob/main/src/starfleet/configuration_files/configuration.yaml) file that contains 3 stanzas, one for `STARFLEET`, one for the `AccountIndexGeneratorShip`, and one for the `StarfleetDefaultAccountIndex`. 6 | The included file is very heavily documented and it should be self-explanatory. The configuration file is set up to conform to what is included in the provided AWS SAM template. 7 | 8 | 1. Open the `src/starfleet/configuration_files/configuration.yaml` file in your favorite text editor. 9 | 1. Go through and update the values of that file accordingly. For the `TemplatePrefix` under the `AccountIndexGeneratorShip` we are going to make that file in the next section. The default value set is perfect: `AccountIndexGenerator/SaveAccountInventory.yaml`. 10 | 1. Save the changes 11 | 12 | At this point you should now have: 13 | 14 | - [x] Enable AWS Organizations if you haven't already done so and move some accounts into it 15 | - [x] Pick out an AWS account for deploying a testing version of Starfleet 16 | - [x] Work on getting a read-only Starfleet IAM role deployed with the permissions outlined above in all your AWS accounts. This role is _not_ very permissive and is only able to describe the enabled regions for an account. 17 | - [x] In the organization root, it has permissions to list the Organizations accounts. 18 | - [x] If you use StackSets then you need to manually make the role in the org root since StackSets won't operate in the org root. 19 | - [x] Important: Make sure that you have some IAM principal that you can use locally that can assume all these roles. This will be needed to run the Starfleet CLI. If you use AWS SSO, then use the ARN for the permissions set provisioned administrative role in the Starfleet account. See the note above for an example. 20 | - [x] Starfleet worker IAM roles deployed everywhere 21 | - [x] And now: the `configuration.yaml` file in `src/starfleet/configuration_files` modified with values unique to your environment 22 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/github_sync/test_schemas.py: -------------------------------------------------------------------------------- 1 | """Tests for the GitHub Sync worker's schemas 2 | 3 | Tests out the schemas for configuration and payload to make sure they are correct. 4 | 5 | :Module: starfleet.tests.starfleet_included_plugins.github_sync.test_schemas 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import pytest 12 | import yaml 13 | from marshmallow import ValidationError 14 | 15 | 16 | def test_payload_template() -> None: 17 | """This tests that the GitHubSyncPayloadTemplate schema has proper validation logic.""" 18 | from starfleet.worker_ships.plugins.github_sync.ship import GitHubSyncPayloadTemplate 19 | 20 | # Good: 21 | payload = """ 22 | TemplateName: SyncRepo 23 | TemplateDescription: Syncs a repo on GitHub with S3 24 | Organization: gemini-oss 25 | Repository: starfleet 26 | BranchName: main 27 | GitHubAppId: "0123456" 28 | GitHubInstallationId: "0123456" 29 | BucketName: some-bucket 30 | BucketRegion: us-east-2 31 | KeyPrefix: Some/Path 32 | ExtractZipContents: True 33 | IncludeRepoPaths: 34 | - "starfleet/*" 35 | ExcludeRepoPaths: 36 | - "*.toml" 37 | DeleteMissingFiles: True 38 | """ 39 | assert GitHubSyncPayloadTemplate().load(yaml.safe_load(payload)) == { 40 | "extract_zip_contents": True, 41 | "bucket_region": "us-east-2", 42 | "key_prefix": "Some/Path", 43 | "template_name": "SyncRepo", 44 | "organization": "gemini-oss", 45 | "bucket_name": "some-bucket", 46 | "exclude_repo_paths": ["*.toml"], 47 | "github_app_id": "0123456", 48 | "delete_missing_files": True, 49 | "github_installation_id": "0123456", 50 | "repository": "starfleet", 51 | "branch_name": "main", 52 | "template_description": "Syncs a repo on GitHub with S3", 53 | "include_repo_paths": ["starfleet/*"], 54 | } 55 | 56 | # Test the region validator: 57 | bad_region = yaml.safe_load(payload) 58 | bad_region["BucketRegion"] = "pewpewpew" 59 | with pytest.raises(ValidationError) as exc: 60 | GitHubSyncPayloadTemplate().load(bad_region) 61 | assert exc.value.messages_dict["BucketRegion"][0].startswith("Must be one of: ") 62 | -------------------------------------------------------------------------------- /src/starfleet/utils/secrets.py: -------------------------------------------------------------------------------- 1 | """Starfleet's Secrets Management module 2 | 3 | Starfleet's manager for secrets are here. This consists of a singleton class to obtain the AWS Secrets Manager secrets for this environment's execution. 4 | 5 | :Module: starfleet.utils.secrets 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import json 12 | from typing import Any, Dict 13 | 14 | import boto3 15 | 16 | from starfleet.utils.configuration import STARFLEET_CONFIGURATION 17 | from starfleet.utils.logging import LOGGER 18 | 19 | # pylint: disable=pointless-string-statement 20 | """Mapping of the Secrets Dict: 21 | { 22 | "STARFLEET": {"SlackToken": "the-token-here"}, 23 | "WORKER_NAME": "Any value - this can be a string, nested Dict, etc.", 24 | "..." 25 | } 26 | """ 27 | 28 | 29 | class SecretsConfigurationMissingError(Exception): 30 | """Raised if the Starfleet Configuration is missing the SecretsManager field in the STARFLEET configuration section.""" 31 | 32 | 33 | class SecretsManager: 34 | """This is the main Starfleet secrets management class.""" 35 | 36 | def __init__(self): 37 | """Default constructor""" 38 | self._secrets = None 39 | 40 | def load_secrets(self) -> None: 41 | """ 42 | This will perform the work to load the secret value from AWS Secrets manager. 43 | 44 | Note: If there are exceptions encountered fetching the secret, it will raise up the stack. 45 | """ 46 | # Do we have a secret to manage? 47 | configuration = STARFLEET_CONFIGURATION.config["STARFLEET"].get("SecretsManager") 48 | if not configuration: 49 | raise SecretsConfigurationMissingError() 50 | 51 | LOGGER.debug(f"[🤐] Loading secrets from Secrets Manager ID in Region: {configuration['SecretId']}/{configuration['SecretRegion']}") 52 | 53 | client = boto3.client("secretsmanager", configuration["SecretRegion"]) 54 | loaded = client.get_secret_value(SecretId=configuration["SecretId"]) 55 | self._secrets = json.loads(loaded["SecretString"]) 56 | LOGGER.debug("[🔑] Secrets loaded successfully") 57 | 58 | @property 59 | def secrets(self) -> Dict[str, Any]: 60 | """Fetch the Secret Dictionary. This will load the secrets if not already loaded.""" 61 | if not self._secrets: 62 | self.load_secrets() 63 | 64 | return self._secrets 65 | 66 | 67 | SECRETS_MANAGER = SecretsManager() 68 | -------------------------------------------------------------------------------- /mkdocs/blog/posts/ecr.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2024-04-08 3 | authors: 4 | - mikegrima 5 | categories: 6 | - General Announcement 7 | --- 8 | 9 | # ECR/Dockerized Lambda Support 10 | We made a small update to the docs to highlight deploying Starfleet with a Dockerized Lambda. This was necessary as Starfleet's dependencies made it larger than the `.zip` file size limit for Lambda. 11 | 12 | The main steps to convert to this from a non-ECR set up is to: 13 | 14 | 1. Create the Private ECR repo in the same account/region you have Starfleet deployed in. 15 | 2. Update the `samconfig.toml` file. 16 | 3. Update the SAM template. 17 | 4. Re-build and deploy. 18 | 19 | The Dockerized Lambda is built with the included `Dockerfile`, which should have everything needed and ready to go. You can test that the container builds by running `docker build .` from within the Starfleet directory. 20 | 21 | ## Update SAM Config 22 | Once you create your ECR Repo, you then need to update your `samconfig.toml` file to include the line: 23 | ``` 24 | image_repository = "ACCOUNT_ID.dkr.ecr.REGION.amazonaws.com/REPONAME" 25 | ``` 26 | ... under your `.deploy.parameters` sections. 27 | 28 | ## Update your SAM Template 29 | The SAM template also needs to be updated. Included now are 2 sample SAM Templates: `test_sam_template.yaml`, which is the ECR sample template, and `test_sam_template_NO_ECR.yaml`, which is the original non-ECR version. 30 | 31 | You would need to update all the function entires to change it from: 32 | ```yaml 33 | StarbaseEventBridgeFunction: 34 | Type: AWS::Serverless::Function 35 | Properties: 36 | CodeUri: ./src 37 | Handler: starfleet.starbase.entrypoints.eventbridge_timed_lambda_handler 38 | Runtime: python3.12 39 | ``` 40 | over to: 41 | 42 | ```yaml 43 | StarbaseEventBridgeFunction: 44 | Type: AWS::Serverless::Function 45 | Metadata: 46 | DockerTag: starfleet 47 | DockerContext: ./ 48 | Dockerfile: Dockerfile 49 | Properties: 50 | PackageType: Image 51 | ImageConfig: 52 | Command: 53 | - starfleet.starbase.entrypoints.eventbridge_timed_lambda_handler 54 | ``` 55 | 56 | Update all the Lambda definitions to include the `Metadata` section as shown above (this is the same for all the functions) and update the `Properties` section to have the fields above. Remove the old fields. Note: the `Command` field is where you place the Lambda handler path - this is unique to each Lambda function. 57 | 58 | For more information see [the ECR setup documentation](../../installation/SetupECR.md) and also [AWS's documentation here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html). 59 | -------------------------------------------------------------------------------- /tests/worker_ship_utils/testing_plugins/basic_plugin/__init__.py: -------------------------------------------------------------------------------- 1 | """A sample worker plugin. 2 | 3 | Sample worker plugin for unit testing purposes. 4 | 5 | :Module: starfleet.tests.worker_ship_utils.testing_plugins.basic_plugin 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import click 12 | 13 | from starfleet.worker_ships.ship_schematics import StarfleetWorkerShip, WorkerShipBaseConfigurationTemplate 14 | from starfleet.worker_ships.base_payload_schemas import WorkerShipPayloadBaseTemplate 15 | 16 | 17 | @click.group() 18 | def testing_plugin() -> None: 19 | """This is the main group for testing the logic for the test plugin.""" 20 | 21 | 22 | @testing_plugin.command() 23 | def test_command_one(): 24 | """Command to test CLIs""" 25 | 26 | 27 | @testing_plugin.command() 28 | def test_command_two(): 29 | """Command to test CLIs""" 30 | 31 | 32 | @click.group() 33 | def testing_plugin_group_two() -> None: 34 | """This is a second group for testing the logic for the test plugin.""" 35 | 36 | 37 | @testing_plugin_group_two.command() 38 | def group_two_command_one(): 39 | """Command to test CLIs""" 40 | 41 | 42 | @testing_plugin_group_two.command() 43 | def group_two_command_two(): 44 | """Command to test CLIs""" 45 | 46 | 47 | class TestingStarfleetWorkerPlugin(StarfleetWorkerShip): 48 | """Testing Starfleet worker plugin.""" 49 | 50 | configuration_template_class = WorkerShipBaseConfigurationTemplate 51 | payload_template_class = WorkerShipPayloadBaseTemplate 52 | 53 | def execute(self, commit: bool = False) -> None: 54 | """This will execute the job from the payload.""" 55 | raise NotImplementedError("pew pew pew") # pragma: no cover 56 | 57 | 58 | class TestingStarfleetWorkerPluginTwo(TestingStarfleetWorkerPlugin): 59 | """A second testing Starfleet worker plugin.""" 60 | 61 | def execute(self, commit: bool = False) -> None: 62 | """This will execute the job from the payload.""" 63 | raise NotImplementedError("pew pew pew") # pragma: no cover 64 | 65 | 66 | class TestingStarfleetWorkerPluginThree(TestingStarfleetWorkerPlugin): 67 | """A second testing Starfleet worker plugin.""" 68 | 69 | def execute(self, commit: bool = False) -> None: 70 | """This will execute the job from the payload.""" 71 | raise NotImplementedError("pew pew pew") # pragma: no cover 72 | 73 | 74 | WORKER_SHIP_PLUGINS = [TestingStarfleetWorkerPlugin, TestingStarfleetWorkerPluginTwo, TestingStarfleetWorkerPluginThree] 75 | CLICK_CLI_GROUPS = [testing_plugin, testing_plugin_group_two] 76 | -------------------------------------------------------------------------------- /tests/test_cli_components.py: -------------------------------------------------------------------------------- 1 | """Tests for Starfleet's CLI components 2 | 3 | Verifies that the CLI components are functioning properly. 4 | 5 | :Module: starfleet.tests.test_cli_components 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | from typing import Any, Dict 12 | 13 | # pylint: disable=unused-argument 14 | import click 15 | from click.testing import CliRunner 16 | 17 | from starfleet.cli.components import StarfleetCliLoader, StarfleetClickGroup 18 | from starfleet.worker_ships.loader import StarfleetWorkerShipLoader 19 | 20 | 21 | def test_cli_startup(test_worker_ship_loader: StarfleetWorkerShipLoader, test_cli_loader: StarfleetCliLoader, test_configuration: Dict[str, Any]) -> None: 22 | """This tests the main CLI startup entrypoint to load the testing worker ships. 23 | 24 | This also tests most of the CLI loader at the same time. 25 | """ 26 | runner = CliRunner() 27 | 28 | assert test_configuration["STARFLEET"]["SlackEnabled"] # This should be set to enabled before the CLI runs. 29 | 30 | # This needs to be defined here to guarantee that the main worker ship and CLI loading doesn't take place prior to mocking 31 | # (need to mock out both before this loads): 32 | @click.group(cls=StarfleetClickGroup) 33 | def cli_group_testing() -> None: 34 | """A CLI group for testing""" 35 | 36 | result = runner.invoke(cli_group_testing) # noqa 37 | assert result.exit_code == 0 38 | 39 | # After the CLI runs we should not have Slack enabled: 40 | assert not test_configuration["STARFLEET"]["SlackEnabled"] 41 | 42 | assert len(cli_group_testing.commands["testing-plugin"].commands) == 2 # noqa 43 | assert len(cli_group_testing.commands["testing-plugin-group-two"].commands) == 2 # noqa 44 | assert cli_group_testing.commands["testing-plugin"].commands["test-command-one"] # noqa 45 | assert cli_group_testing.commands["testing-plugin"].commands["test-command-two"] # noqa 46 | assert cli_group_testing.commands["testing-plugin-group-two"].commands["group-two-command-one"] # noqa 47 | assert cli_group_testing.commands["testing-plugin-group-two"].commands["group-two-command-two"] # noqa 48 | 49 | 50 | def test_main_cli() -> None: 51 | """This tests that the main CLI can load successfully.""" 52 | from starfleet.cli.entrypoint import cli 53 | 54 | runner = CliRunner() 55 | result = runner.invoke(cli) # noqa 56 | assert result.exit_code == 0 57 | assert "Starfleet is a totally awesome whole-infrastructure automation tool." in result.output 58 | 59 | # Not testing the loaded plugins since that will depend on what is or is not enabled in the configuration. 60 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/niceties.py: -------------------------------------------------------------------------------- 1 | """A general set of niceties that Starfleet workers can use to do things that are nice. 2 | 3 | This mostly defines some shortcut code utilities that workers can use for a variety of use cases. 4 | 5 | :Module: starfleet.worker_ships.niceties 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import datetime 12 | import json 13 | from typing import Any 14 | from urllib.parse import unquote_plus 15 | 16 | 17 | def un_wrap_json(json_obj: Any) -> Any: 18 | """Helper function to unwrap nested JSON in the AWS Config resource configuration.""" 19 | # pylint: disable=C0103,W0703,R0911 20 | # Is this a field that we can safely return? 21 | if isinstance(json_obj, (type(None), int, bool, float)): # noqa 22 | return json_obj 23 | 24 | # Is this a Datetime? Convert it to a string and return it: 25 | if isinstance(json_obj, datetime.datetime): 26 | return str(json_obj) 27 | 28 | # Is this a Dictionary? 29 | if isinstance(json_obj, dict): 30 | decoded = {} 31 | for k, v in json_obj.items(): 32 | decoded[k] = un_wrap_json(v) 33 | 34 | # Is this a List? 35 | elif isinstance(json_obj, list): 36 | decoded = [] 37 | for x in json_obj: 38 | decoded.append(un_wrap_json(x)) 39 | 40 | # Yes, try to sort the contents of lists. This is because AWS does not consistently store list ordering for many resource types: 41 | try: 42 | sorted_list = sorted(decoded) 43 | decoded = sorted_list 44 | except Exception: # noqa # nosec # If we can't sort then NBD 45 | pass 46 | else: 47 | # Try to load the JSON string: 48 | try: 49 | # Check if the string starts with a "[" or a "{" (because apparently '123' is a valid JSON 😒😒😒) 50 | for check_field in ["{", "[", '"{', '"[']: # Some of the double-wrapping is really ridiculous 😒 51 | if json_obj.startswith(check_field): 52 | decoded = json.loads(json_obj) 53 | 54 | # If we loaded this properly, then we need to pass the decoded JSON back in for all the nested stuff: 55 | return un_wrap_json(decoded) 56 | 57 | # Check if this string is URL Encoded - if it is, then re-run it through: 58 | decoded = unquote_plus(json_obj) 59 | if decoded != json_obj: 60 | return un_wrap_json(decoded) 61 | 62 | return json_obj 63 | 64 | # If we didn't get a JSON back (exception), then just return the raw value back: 65 | except Exception: # noqa 66 | return json_obj 67 | 68 | return decoded 69 | -------------------------------------------------------------------------------- /mkdocs/developerGuide/primaryComponents/SecretsManager.md: -------------------------------------------------------------------------------- 1 | # Secrets Manager 2 | Starfleet includes a singleton component for interacting with AWS Secrets Manager. This allows you to securely store sensitive details that Starfleet workers can access. 3 | 4 | ## Secrets Format 5 | The secrets are stored as a JSON string in Secrets Manager. It generally follows this format: 6 | 7 | ```json 8 | { 9 | "STARFLEET": { 10 | "SlackToken": "the-token-here" 11 | }, 12 | "WORKER_NAME": "Any value - this can be a string, nested Dict, etc.", 13 | "...More Workers Here..." 14 | } 15 | ``` 16 | 17 | The secret entire secret as shown above is stored as a string in Secrets Manager. A singleton, `SECRETS_MANAGER` resides in `starfleet.utils.secrets`, and it will lazy load the secrets when you reference the `SECRETS_MANAGER.secrets` property, which is a simple Python dictionary. 18 | 19 | You will also note the `SlackToken` above under the `STARFLEET` section of the JSON. That is where the Slack token is used for Slack alerts. Here is a small code snippet of the Slack component referencing the Slack token secret: 20 | 21 | 22 | ```python 23 | if not self._web_client: 24 | self._web_client = WebClient(token=SECRETS_MANAGER.secrets["STARFLEET"]["SlackToken"]) 25 | ``` 26 | 27 | You simply reference the secret out of the dictionary in the same way that you would for the Starfleet configuration. 28 | 29 | 30 | ## Configuration Requirement 31 | If you are making use of the Secrets Management component, you have to make sure that the Starfleet configuration is configured to point to it. Under the main `STARFLEET` stanza of the configuration you need to have the following: 32 | 33 | ```yaml 34 | SecretsManager: 35 | SecretId: Starfleet # This is the name of the AWS Secrets Manager secret. The secret must reside in the same AWS account as Starfleet. 36 | SecretRegion: us-east-2 # This is the AWS region for where the secret resides. 37 | ``` 38 | 39 | !!! note "IAM Permissions" 40 | If you make use of the secret, you will need to make sure that the Starfleet worker in question has IAM permissions to access the secret. See the included SAM template for an example of what this should look like. 41 | 42 | !!! note "Cached Data Notice" 43 | Like the other singletons in Starfleet, the loaded data will persist in memory on subsequent runs of the Lambda function. This has the benefit of not needing to make repeated AWS Secrets Manager API calls, which saves money. The downside to this is that AWS will persist the secrets value in memory as long as the container running your Lambda function remains operational (this happens on the AWS backend; you don't have control over it). If you make an update to the secret string, subsequent Lambda calls may not yet see it and you could operate off of the old cached secrets value. 44 | 45 | AWS will periodically rotate the Lambda container out every few hours if no code has changed. The only way for you to force your Lambda function's container to rotate out is to actually update the code for your Lambda function. 46 | -------------------------------------------------------------------------------- /mkdocs/developerGuide/CheckList.md: -------------------------------------------------------------------------------- 1 | # Development Check List 2 | 3 | This page is a simple check list to double-check that you have done all the steps you needed. 4 | 5 | - [x] You set up the virtual environment and are inside of the virtual environment - [Link](Overview.md#set-up-virtual-environment) 6 | - [x] You installed the requirements - [Link](Overview.md#install-the-dependencies) 7 | - [x] You created (or your deployment script places) your Worker Ship Python package under `src/starfleet/worker_ships/plugins/` - [Link](primaryComponents/workerShips/Overview.md#worker-ship-residency) 8 | - [x] Your Worker Ship Plugin has a configuration schema that is a subclass of `WorkerShipBaseConfigurationTemplate` - [Link](primaryComponents/workerShips/Overview.md#configuration) 9 | - [x] Your Worker Ship Plugin has a payload template schema that is a subclass of `WorkerShipPayloadBaseTemplate` (this includes the account and account/region base classes) - [Link](primaryComponents/workerShips/Overview.md#payload-template) 10 | - [x] Your Worker Ship Plugin contains a class that subclasses `StarfleetWorkerShip` _and_ configures it with all the required components, like the schemas, and also implements the `execute` function to do the thing that needs to be done - [Link](primaryComponents/workerShips/Overview.md#the-worker-ship-class) 11 | - [x] You created a `lambda_handler` function that is wrapped by the `@worker_lambda` decorator - [Link](primaryComponents/workerShips/LambdaEntrypoints.md) 12 | - [x] You created a `click.group` decorated function, and also some CLI commands for that group - [Link](primaryComponents/workerShips/CLI.md) 13 | - [x] In your plugin package's `__init__.py`, you defined both `WORKER_SHIP_PLUGINS` set to a list of the `StarfleetWorkerShip` classes you defined - [Link](primaryComponents/workerShips/Loader.md#make-starfleet-see-your-worker-and-clis) 14 | - [x] In your plugin package's `__init__.py`, you defined both `CLICK_CLI_GROUPS` set to a list of the `click.group()` decorated functions you defined - [Link](primaryComponents/workerShips/Loader.md#make-starfleet-see-your-worker-and-clis) 15 | - [x] You made extensive pytest tests for your Worker Ship Plugin with nice fixtures 100% test coverage. Yes 💯% test coverage! See the existing tests for details on how to make good tests. 16 | - [x] You created (or your deployment script places) your Starfleet configuration with the proper configuration entries in `src/starfleet/configuration` - [Link](primaryComponents/ConfigurationManager.md) 17 | - [x] You made the necessary changes to the SAM Template for your Lambdas to get deployed (don't forget to update the configuration to include the SQS URLs!) - [Link](SAMConfiguration.md) 18 | - [x] Your payload template resides in the template S3 bucket where your worker expects it 19 | - [x] Make sure your Lambda has enough time and memory to run. You'll need to monitor the logs to see how much RAM and time it takes to run your workload to make adjustments in the SAM template. 20 | 21 | There are probably more but this should help you isolate and detect issues should they arise. 22 | -------------------------------------------------------------------------------- /mkdocs/architecture/Overview.md: -------------------------------------------------------------------------------- 1 | # Starfleet Architecture Overview 2 | This page discusses the components in the Starfleet architecture and what they do. 3 | 4 | ## Note: Many components are still in progress of being implemented 5 | 6 | ## Diagram 7 | ![Starfleet Architecture](../images/StarfleetArchitecture.svg) 8 | 9 | ## General Gist 10 | The general gist is as follows: 11 | 12 | * There is 1 AWS account that houses all the Starfleet components 13 | * Lambda functions (called worker "ships") execute tasks in your environment with a payload that provides context on what to do 14 | * An IAM role exists in all your AWS accounts that the worker "ship" Lambda functions can assume so it can operate in the AWS account in question 15 | * The Lambda functions are tasked by a component called the `Starbase` - this generates the payload and tasks the worker ship to execute 16 | * An AWS account index/inventory exists to inform the `Starbase` on which accounts need to be tasked 17 | * Serverless and idempotent; highly parallelized 18 | 19 | ## Components 20 | 21 | 1. **Worker Ship Lambdas** - These are Lambda functions that go out and actually perform the desired workload. Worker ship Lambdas are tasked from an SQS queue with the payload describing the workload to be performed in the account (or account/region) in question. The Starbase generates the payload for the given worker ship and places it in the worker's SQS queue. Each worker ship also has a unique configuration that can be set. 22 | 1. **Payload Template YAML files** - These are the YAML templates that will be provided to the worker ships to describe what an infrastructure state should look like. The worker is supposed to use the template to know what actions to be performed in the target destination. 23 | 1. **Starbase** - This is a Lambda function that tasks workers for jobs to run. 24 | 1. **Payload Template S3 Bucket** - This is an S3 bucket that holds the payload template YAMLs. This bucket is configured to have notifications (to be implemented) that invoke the Starbase. The Starbase is then able to task the desired worker ship with the template payload to execute. The event system allows for CI/CD. 25 | 1. **AWS Account Index** - This is an index or inventory of all AWS accounts that Starfleet should operate on. By default, Starfleet ships with a worker ship that automatically generates a JSON file based on AWS Organizations. This JSON file is used by the Starbase to know which accounts exist and when to task the worker ships. 26 | 1. **EventBridge Time Based events** - The Starbase can be invoked with EventBridge timed events. This allows for CRON-like invocation of workers. Worker ships can be configured to get invoked on a time schedule. 27 | 1. **Resident IAM role** - This is an IAM role that exists in all AWS accounts that Starfleet operates in. This role (or roles) allows the Starfleet worker ships to assume into a destination account and then operate within it. Starfleet uses a hub-spoke type of model (octopus) to perform tasks in your infrastructure. _**Note: Starfleet is a security sensitive application and as such should reside in a limited-access AWS account!**_ 28 | 1. **Dedicated worker SQS queues** - Not yet implemented - these are SQS queues (or topics?) that will be used to directly invoke a worker ship through the Starbase. 29 | 30 | ## Next Sections 31 | The next sections go into detail about what each component does and why. 32 | -------------------------------------------------------------------------------- /src/starfleet/utils/plugin_loader.py: -------------------------------------------------------------------------------- 1 | """Starfleet's worker ship plugin loader logic. 2 | 3 | This does all the logic required to load Starfleet plugins. 4 | 5 | :Module: starfleet.util.plugin_loader 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | # from importlib.metadata import entry_points 12 | import importlib 13 | from pkgutil import iter_modules 14 | from typing import Dict, List, Type 15 | 16 | from starfleet.utils.logging import LOGGER 17 | 18 | 19 | class InvalidPluginListException(Exception): 20 | """Exception raised if the name for where to pull out a list of plugins isn't actually a list.""" 21 | 22 | 23 | class InvalidPluginClassException(Exception): 24 | """Exception raised if the plugin is not a valid Starfleet plugin object.""" 25 | 26 | 27 | def find_plugins(package_path: str, package_prefix: str, plugin_attr_name: str, plugin_super_class: Type, verify_class=True) -> Dict[str, List[Type]]: 28 | """This is a function that will be used for loading Starfleet plugins. It will work by iterating for plugins that reside in packages in the given path. This will then 29 | attempt to load the modules for that path. 30 | 31 | For all Starfleet plugins, there needs to be an __init__.py that specifies the `plugin_attr_name` that is passed in here, which is a list of the plugin classes to load. 32 | There is also a `plugin_super_class`, which is the superclass that the plugin must be a subclass of (`verify_class` = True) -- OR -- 33 | it will check if the object is an instance of the super class -- this is set by the boolean flag passed in as `verify_class` = False 34 | """ 35 | plugins = {} 36 | if verify_class: 37 | verify_method = issubclass 38 | else: 39 | verify_method = isinstance 40 | 41 | LOGGER.debug(f"[🏗️] Loading plugins in the {package_path} location...") 42 | for package in iter_modules(package_path, package_prefix): 43 | LOGGER.debug(f"[⚙️] Processing module named: {package.name}") 44 | module = importlib.import_module(package.name) 45 | 46 | # Next, check that the module contains the variable that will hold the plugin classes: 47 | if hasattr(module, plugin_attr_name): 48 | # Next, check that it's a list: 49 | list_of_plugins = getattr(module, plugin_attr_name) 50 | if not isinstance(list_of_plugins, list): 51 | raise InvalidPluginListException( 52 | f"[💥] The package: {package.name} needs a variable named {plugin_attr_name} that is of type List, not " f"type: {type(list_of_plugins)}" 53 | ) 54 | 55 | for plugin in list_of_plugins: 56 | if not verify_method(plugin, plugin_super_class): 57 | raise InvalidPluginClassException( 58 | f"[💥] The plugin: {plugin.__name__} in package: {package.name} does not properly subclass: {plugin_super_class.__name__}" 59 | ) 60 | 61 | plugins[package.name] = list_of_plugins 62 | LOGGER.debug(f"[👍] Found {len(list_of_plugins)} plugins in {package_path}") 63 | 64 | else: 65 | LOGGER.debug(f"[⏭️] Skipping module: {package.name} because it lacks the {plugin_attr_name} List") 66 | 67 | return plugins 68 | -------------------------------------------------------------------------------- /src/starfleet/cli/components.py: -------------------------------------------------------------------------------- 1 | """Components for the CLI to make it function properly. 2 | 3 | These are pulled out here to make it easy to test and avoid circular dependencies. 4 | 5 | :Module: starfleet.cli.components 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | from typing import Any, List 12 | 13 | import click 14 | 15 | import starfleet.worker_ships.plugins 16 | from starfleet.utils.configuration import STARFLEET_CONFIGURATION 17 | from starfleet.worker_ships.loader import STARFLEET_WORKER_SHIPS 18 | from starfleet.startup import base_start_up 19 | from starfleet.utils.logging import LOGGER 20 | from starfleet.utils.plugin_loader import find_plugins 21 | 22 | 23 | LOGO = """ 24 | _/ _/_/ _/ _/ 25 | _/_/_/ _/_/_/_/ _/_/_/ _/ _/_/ _/ _/ _/_/ _/_/ _/_/_/_/ 26 | _/_/ _/ _/ _/ _/_/ _/_/_/_/ _/ _/_/_/_/ _/_/_/_/ _/ 27 | _/_/ _/ _/ _/ _/ _/ _/ _/ _/ _/ 28 | _/_/_/ _/_/ _/_/_/ _/ _/ _/ _/_/_/ _/_/_/ _/_/ 29 | """ 30 | 31 | 32 | class StarfleetCliLoader: 33 | """This will locate all the CLIs.""" 34 | 35 | # These are defined here for easy testability -- this is the same path to the worker ships: 36 | _worker_ship_path: str = starfleet.worker_ships.plugins.__path__ 37 | _worker_ship_prefix: str = starfleet.worker_ships.plugins.__name__ + "." 38 | 39 | def __init__(self): 40 | self._clis: List[click.Group] = None # noqa 41 | 42 | def load_clis(self): 43 | """This will load all Starfleet worker ship plugins and verify that they are set up properly. This is code that will mostly be used by both the 44 | Starbase and the CLI.""" 45 | LOGGER.debug("[🖥️] Loading CLIs (which are just plugins)...") 46 | self._clis = [] 47 | for _, cli_list in find_plugins(self._worker_ship_path, self._worker_ship_prefix, "CLICK_CLI_GROUPS", click.Group, verify_class=False).items(): 48 | # Avoid too much debug output here... 49 | self._clis.extend(cli_list) 50 | LOGGER.debug(f"[🖥️] Completed loading {len(self._clis)} CLIs") 51 | 52 | @property 53 | def clis(self) -> List[click.Group]: 54 | """Gets the CLIs and lazy-loads them if not already set.""" 55 | if self._clis is None: 56 | self.load_clis() 57 | 58 | return self._clis 59 | 60 | 61 | STARFLEET_CLI_LOADER = StarfleetCliLoader() # Not really needed to be defined here, but it makes unit testing 1000x easier. 62 | 63 | 64 | class StarfleetClickGroup(click.Group): 65 | """The Starfleet Click Group. This is here to print the logo :D""" 66 | 67 | def __init__(self, **attrs: Any): 68 | super().__init__(**attrs) 69 | 70 | # Print out the awesome logo: 71 | click.echo(LOGO) 72 | 73 | # Base start up: 74 | base_start_up() 75 | 76 | # Load the worker ships (makes sure everything is all good): 77 | STARFLEET_WORKER_SHIPS.get_worker_ships() 78 | 79 | # Disable Slack alerts: 80 | STARFLEET_CONFIGURATION.config["STARFLEET"]["SlackEnabled"] = False 81 | 82 | # Load up the CLIs: 83 | for command in STARFLEET_CLI_LOADER.clis: 84 | self.add_command(command) 85 | -------------------------------------------------------------------------------- /mkdocs/userGuide/Overview.md: -------------------------------------------------------------------------------- 1 | # User Guide 2 | This page outlines how to use Starfleet and provides a guide around using each included worker ship. 3 | 4 | As previously mentioned, the purpose of Starfleet is to augment AWS Organizations. It should be used to enable features that you need everywhere to help maintain a secure cloud infrastructure state. 5 | 6 | In this section of the documentation, we provide a general overview of how to use each included plugin, as well as some common troubleshooting. 7 | 8 | ## General Gist 9 | In general Starfleet works based on a combination of: 10 | 11 | 1. The Starbase 12 | 1. Deployed Workers 13 | 1. Configuration 14 | 1. Payloads 15 | 16 | Each worker has its own Configuration and Payload definitions. You should be familiar with the [Architecture](../architecture/Overview.md) and [Installation Guide](../installation/Overview.md) before moving on to this section because it will make a lot of the topics discussed familiar. 17 | 18 | ## General Troubleshooting 19 | The Starbase and the worker ship plugins are Lambda functions and will output logs to CloudWatch LogGroups. Reviewing the logs in the log group will provide you with the best details on what's happening under the hood. 20 | 21 | ## Command Line Interface 22 | Starfleet has a command line interface that is executable via the `starfleet` command once you are in an activate virtual environment. All plugins should expose some command groups that you can execute and then explore for additional details. 23 | 24 | In general, you will need AWS credentials for Starfleet to utilize to ensure that these commands can run. This is mostly documented in the [Installation Guide](../installation/IAM.md). 25 | 26 | Each worker ship should have documentation on how to execute the respective commands. 27 | 28 | ### Common CLI Components 29 | In general all worker ships utilize common components to expose their CLIs. All worker ships will have a the following fields in common: 30 | 31 | #### Payload (`--payload`) 32 | The payload flag (`--payload`) specifies where to load the payload YAML (`--payload path/to/payload/template`) for the worker ship CLI. This is used as follows: 33 | 34 | ```bash 35 | starfleet account-index generate --payload some/path/to/your/payload.yaml 36 | ``` 37 | 38 | #### The Commit Flag (`--commit`) 39 | The commit flag (`--commit`) must be added to inform the worker ship plugin that it should operate in a commit mode to make changes. Not including the flag will ensure that the worker ship runs in read-only mode. 40 | 41 | Omitting the `--commit` flag is analogous to running `terraform plan`, and adding the `--commit` flag is analogous to running `terraform apply`. 42 | 43 | Example of it's usage is: 44 | 45 | ```bash 46 | starfleet account-index generate --payload some/path/to/your/payload.yaml --commit 47 | ``` 48 | 49 | #### Account ID (`--account-id`) 50 | In addition to the above, Account worker ships will also require an `--account-id` argument that takes in the 12-digit AWS Account ID of the AWS account to operate the payload template against. Here is an example of that being used: 51 | 52 | ```bash 53 | starfleet some-account-worker some-command --payload some/path/to/the/payload.yaml --account-id 111111111111 54 | ``` 55 | 56 | #### Region (`--region`) 57 | Account/Region workers need both the `--account-id` and the `--region` flags passed in. The region is the name of the AWS region to operate in. Here is an example: 58 | 59 | ```bash 60 | starfleet aws-config sync --payload some/path/to/the/payload.yaml --account-id 111111111111 --region us-east-1 61 | ``` 62 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/iam/conftest.py: -------------------------------------------------------------------------------- 1 | """Pytest fixtures for the IAM worker ships 2 | 3 | All the Pytest fixtures unique to the AWS IAM workers 4 | 5 | :Module: starfleet.tests.starfleet_included_plugins.iam.conftest 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | # pylint: disable=unused-argument,unused-import,redefined-outer-name 12 | import os 13 | from typing import Any, Dict, Generator 14 | 15 | import boto3 16 | import pytest 17 | import yaml 18 | from botocore.client import BaseClient 19 | from moto import mock_aws 20 | 21 | from tests.account_index.conftest import test_index # noqa 22 | 23 | 24 | @pytest.fixture 25 | def aws_iam(aws_credentials: None) -> Generator[BaseClient, None, None]: 26 | """This is a fixture for a Moto wrapped AWS IAM mock for the entire unit test.""" 27 | os.environ["MOTO_ACCOUNT_ID"] = "000000000001" 28 | os.environ["MOTO_IAM_LOAD_MANAGED_POLICIES"] = "true" 29 | 30 | with mock_aws(): 31 | yield boto3.client("iam", region_name="us-east-1") 32 | 33 | del os.environ["MOTO_ACCOUNT_ID"] 34 | del os.environ["MOTO_IAM_LOAD_MANAGED_POLICIES"] 35 | 36 | 37 | @pytest.fixture 38 | def test_role(aws_iam: BaseClient, aws_sts: BaseClient) -> None: 39 | """This is a sample IAM role that is used for testing.""" 40 | aws_iam.create_role( 41 | RoleName="StarfleetIambicTesting", 42 | AssumeRolePolicyDocument='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}', 43 | Description="This is a test role", 44 | ) 45 | 46 | 47 | @pytest.fixture() 48 | def template() -> Dict[str, Any]: 49 | """Template for describing what the test role should be""" 50 | return yaml.safe_load( 51 | """ 52 | TemplateName: TestingTemplate 53 | TemplateDescription: This is a test template IAM Role 54 | IncludeAccounts: 55 | AllAccounts: True 56 | OperateInOrgRoot: True 57 | IambicVariables: 58 | - Key: some_key 59 | Value: some_value 60 | - Key: some_other_key 61 | Value: some_other_value 62 | IambicRoleTemplate: 63 | properties: 64 | description: 'Starfleet iambic test role with variable {{var.some_key}}' 65 | assume_role_policy_document: 66 | statement: 67 | - action: sts:AssumeRole 68 | effect: Allow 69 | principal: 70 | service: ec2.amazonaws.com 71 | version: '2012-10-17' 72 | managed_policies: 73 | - policy_arn: arn:aws:iam::aws:policy/ReadOnlyAccess 74 | inline_policies: 75 | - policy_name: 'SomePolicyIn-{{var.account_name}}' 76 | StarfleetIncludeAccounts: 77 | ByOrgUnits: 78 | - SomeNestedOU 79 | StarfleetExcludeAccounts: 80 | ByNames: 81 | - Account 10 82 | statement: 83 | - effect: Deny 84 | action: s3:* 85 | resource: '*' 86 | version: '2012-10-17' 87 | role_name: StarfleetIambicTesting 88 | tags: 89 | - key: owner 90 | value: pewpewpew 91 | - key: some_other_key 92 | value: '{{var.some_other_key}}' 93 | """ 94 | ) 95 | -------------------------------------------------------------------------------- /mkdocs/installation/PreparePayload.md: -------------------------------------------------------------------------------- 1 | # Prepare a Payload Template 2 | 3 | We are going to create a payload template for the `AccountIndexGeneratorShip` plugin so that we can generate the account index. 4 | 5 | !!! note "Important Note for Now" 6 | Starfleet will have a chicken and egg problem: it requires an account index to function. But there is also a worker that makes an account index. The account index worker can't run unless an account index exists. 7 | 8 | We will solve that problem by running the CLI to generate that index after we perform a deployment. For now don't worry about this and just focus on making a payload template. All of this will come together in the next sections! We promise! :) 9 | 10 | 1. Open your favorite text editor and you are going to create a new file. We will revisit this file so keep it somewhere you can reference to later. 11 | 1. We are going to name this file: `SaveAccountInventory.yaml` 12 | 1. In this file, copy and paste in the following, which is a payload template that will inform the `AccountIndexGeneratorShip` with additional details on how and where to save the account index: 13 | ```yaml 14 | TemplateName: AccountIndexGeneratorShip 15 | TemplateDescription: The Account Index Generator Worker Ship template 16 | AccountInventoryBucket: starfleet-account-index-DEPLOYMENT-ACCOUNT-ID 17 | InventoryBucketRegion: DEPLOYMENT-REGION 18 | ``` 19 | 1. In that file, replace the `DEPLOYMENT-ACCOUNT-ID` and `DEPLOYMENT-REGION` with the AWS account ID for your deployment and region, and save the file. 20 | 21 | !!! note "Why do we need the payload template for this?" 22 | In case you are wondering "why" we need a payload template for this worker, the payload template tells the worker which S3 bucket to save the index to. It also tells the worker and what to call the index file (by default it uses `accountIndex.json`). This makes this super flexible. Let's say you want to dump multiple indexes to multiple S3 buckets and prefixes. You can do that! Just make a separate payload template to describe that action and it will be done! 23 | 24 | However, for now, we are just setting this up to be consistent with what the AWS SAM template is doing, which is going to address 99.99% of use cases. 25 | 26 | At this point you should now have: 27 | 28 | - [x] Enable AWS Organizations if you haven't already done so and move some accounts into it 29 | - [x] Pick out an AWS account for deploying a testing version of Starfleet 30 | - [x] Work on getting a read-only Starfleet IAM role deployed with the permissions outlined above in all your AWS accounts. This role is _not_ very permissive and is only able to describe the enabled regions for an account. 31 | - [x] In the organization root, it has permissions to list the Organizations accounts. 32 | - [x] If you use StackSets then you need to manually make the role in the org root since StackSets won't operate in the org root. 33 | - [x] Important: Make sure that you have some IAM principal that you can use locally that can assume all these roles. This will be needed to run the Starfleet CLI. If you use AWS SSO, then use the ARN for the permissions set provisioned administrative role in the Starfleet account. See the note above for an example. 34 | - [x] AWS Account identified for deployment 35 | - [x] Starfleet worker IAM roles deployed everywhere 36 | - [x] The `configuration.yaml` file in `src/starfleet/configuration_files` modified with values unique to your environment 37 | - [x] And now: a payload template (not stored as a configuration file) in a different place than your configuration that describes what the Starfleet Account Index Generator is supposed to do 38 | -------------------------------------------------------------------------------- /mkdocs/installation/Overview.md: -------------------------------------------------------------------------------- 1 | # Installation Guide 2 | 3 | !!! warning 4 | Starfleet is a very sophisticated tool geared towards security and infrastructure engineers with AWS and software development experience. A lot of the instructions are geared towards that audience and in some cases, you may need to roll up your sleeves. 5 | 6 | This page is a guide for installing Starfleet. Our instructions are generally geared around [AWS SAM](https://aws.amazon.com/serverless/sam/). If you haven't already, please install the AWS SAM CLI. 7 | 8 | There are other components that are not covered by SAM, like the Starfleet IAM role, which is also discussed. 9 | 10 | ## Included components 11 | Separate docs exist for each included worker ship, but for the purposes of installation, the following are the minimum installed components: 12 | 13 | 1. Starfleet account resident IAM roles 14 | 1. The Starbase Lambda Functions 15 | 1. The `AccountIndexGeneratorShip`, which is a worker ship that generates an AWS account inventory by assuming a role in the Organization Root account, listing all the accounts, then fetching specific details about each account, and then finally saves the result as a JSON file to an S3 bucket. 16 | 1. The `StarfleetDefaultAccountIndex`, which is an Account Indexer that relies on the generated JSON file produced by the `AccountIndexGeneratorShip`. 17 | 1. AWS resources like the SQS Queues, the template S3 bucket, the account index bucket, the EventBridge events, and all the Starfleet Lambda IAM roles 18 | 19 | Most of these are created with AWS SAM template, but some sleeve rolling is required for several of the others. 20 | 21 | ## SAM Template Summary 22 | AWS SAM templates are YAML files that are effectively CloudFormation templates with some additional (very nice) abstractions. SAM will also manage the building and uploading of artifacts to S3 so that CloudFormation can properly deploy your Lambdas. 23 | 24 | Included in the base repository directory `starfleet/` is a _sample_ SAM template named [`test_sam_template.yaml`](https://github.com/gemini-oss/starfleet/blob/main/test_sam_template.yaml). The SAM template outlines the primary components, like the SQS queues, the template S3 bucket, the Starbase Lambdas, and the worker Lambda functions. 25 | 26 | However, before deploying anything with AWS SAM, please review the Prerequisites below. 27 | 28 | ## Prerequisites 29 | Before deployment you need to review the following 3 things: 30 | 31 | 1. AWS Organizations is enabled and used 32 | 1. The AWS account for Starfleet deployment 33 | 1. IAM roles used by Starfleet 34 | 35 | ### Starfleet AWS Account(s) 36 | Starfleet is a security sensitive tool with powerful capabilities that can operate across your entire cloud infrastructure. As such, you will want to have this live in a security sensitive account with few co-tenant applications. 37 | 38 | It is also recommended to have an account for a test version of Starfleet. This guide assumes that you have 2 accounts, a separate testing and production account, that you can deploy a testing and production Starfleet to. 39 | 40 | !!! warning 41 | Starfleet is a very privileged application. It is intended to operate over your entire AWS cloud infrastructure. Please place it in an Account with limited access and few other deployed applications. 42 | 43 | Our default account inventory makes the assumption that you are making use of AWS organizations and have your accounts registered with it (seriously, you need to do this if you haven't already). 44 | 45 | Before you can move forward, you will need to do the following: 46 | 47 | - [x] Enable AWS Organizations if you haven't already done so and move some accounts into it 48 | - [x] Pick out an AWS account for deploying a testing version of Starfleet 49 | -------------------------------------------------------------------------------- /mkdocs/userGuide/awsConfig/AWSConfigWorker.md: -------------------------------------------------------------------------------- 1 | # AWS Config Worker Ship User Guide 2 | This is the main user guide for the AWS Config worker ship. This page documents all that there is to know about this worker ship. 3 | 4 | !!! warning "Schema Change June 2023" 5 | The initial schema for this worker was changed in June 2023 to introduce support for AWS Config's resource 6 | exclusion feature in the recorders. See the Payload Templates section for details on what the current schema is. 7 | 8 | ## What does it do? 9 | This is a very important worker ship as it allows you to easily enable [AWS Config](https://aws.amazon.com/config/) to all accounts and regions in your infrastructure. AWS Config will also be used with other workers to act as a cache of AWS resources. This is done to avoid API rate limiting when describing resources. 10 | 11 | ## Why did we make this? 12 | AWS Config is a very important visibility tool in AWS. It is also a very effective cache of resource state that can be queried without causing rate limits for the respective services. However, AWS Config recorder enablement is not simple to do without this tool. AWS Organizations lacks the ability to enable AWS Config recorders across your environment, and more importantly, you will likely need the ability to customize how AWS Config is configured throughout your accounts and regions. 13 | Being able to customize how AWS Config is enabled across your infrastructure is highly desireable. This is because AWS Config bills you for each configuration change that is recorded. If you have an AWS account and region with a lot of changes being made to a given resource type, then you may incur a lot of charges for those resources being recorded. In such a case, you may still want to enable it for the resource types that don't frequently change so you can still have reasonable configuration history coverage. 14 | With this worker, you can have it all, as you can define in one place how AWS Config should be configured, and you can specify account/region specific overrides. Starfleet will task the workers accordingly to implement the changes throughout very quickly. This will also do both, detect and correct any drift that appears with AWS Config recorders. 15 | 16 | ## What does it NOT do? 17 | It's important that we discuss what this does not do, and that is the aggregator setup. That is not performed by this worker, because AWS Organizations does this for you exceptionally well. We strongly recommend that you follow the [instructions here to set up AWS Config aggregators](https://docs.aws.amazon.com/config/latest/developerguide/set-up-aggregator-cli.html#add-an-aggregator-organization-cli) for your entire Organization. 18 | 19 | ## How it works 20 | The AWS Config worker is an `ACCOUNT_REGION` worker, that is to say that it schedules a task for every account and region that the payload template outlines. This will go out and verify that: 21 | 22 | 1. The AWS Config Recorder is configured to the template's spec 23 | 1. The Delivery Channel is configured to the template's spec 24 | 1. The Retention Configuration is configured to the template's spec 25 | 1. The Recorder is either On or Off based on the template's spec 26 | 27 | This worker ship plugin can be invoked on any EventBridge timed event and/or by updating the S3 template. We recommend that at a minimum, you run this once daily. As mentioned above, we do not set up aggregators because AWS Organizations does this very nicely for you. See the documentation above for more details on how to set that up. 28 | 29 | The next sections describe how to configure, set up, and use this worker ship plugin. 30 | 31 | ## Alerting 32 | The Config worker supports alerts to Slack. It will alert on any errors that are encountered during execution. It will also emit `SUCCESS` notices if it makes a change to the AWS Config configuration. 33 | -------------------------------------------------------------------------------- /src/starfleet/account_index/loader.py: -------------------------------------------------------------------------------- 1 | """# Account Index Plugin Loader 2 | 3 | This defines the loader for the account index plugin. This uses the same logic as the other loaders. The loader here will locate 4 | the account index plugins and it will instantiate the plugin that is set by the Starfleet `AccountIndex` configuration field. 5 | That plugin must exist. Once instantiated, then it is accessible by the loader and can be used by anything that needs it. 6 | 7 | :Module: starfleet.account_index.loader 8 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 9 | :License: See the LICENSE file for details 10 | :Author: Mike Grima 11 | """ 12 | 13 | import starfleet.account_index.plugins 14 | from starfleet.account_index.schematics import AccountIndex, AccountIndexInstance 15 | from starfleet.utils.configuration import STARFLEET_CONFIGURATION 16 | from starfleet.utils.logging import LOGGER 17 | from starfleet.utils.plugin_loader import find_plugins 18 | 19 | 20 | class UnknownAccountIndexError(Exception): 21 | """Exception that is raised when a requested account index isn't in the registry.""" 22 | 23 | 24 | class StarfleetAccountIndexLoader: 25 | """This will load the account index plugins and is used by the Starbase to interact with them.""" 26 | 27 | # These are defined here for easy testability: 28 | _index_ship_path: str = starfleet.account_index.plugins.__path__ 29 | _index_ship_prefix: str = starfleet.account_index.plugins.__name__ + "." 30 | 31 | def __init__(self): 32 | self._index: AccountIndexInstance = None # noqa 33 | 34 | def reset(self): 35 | """Used for unit tests. This resets the index.""" 36 | self._index = None 37 | 38 | def load_indexes(self): 39 | """ 40 | This will load all the account index plugins and add them to a registry. Once in the registry the chosen account index class will be 41 | instantiated and accessed. 42 | """ 43 | registry = {} 44 | self._index = None 45 | 46 | LOGGER.debug("[📇] Loading the account index plugins...") 47 | try: 48 | for _, plugin_classes in find_plugins(self._index_ship_path, self._index_ship_prefix, "ACCOUNT_INDEX_PLUGINS", AccountIndex).items(): 49 | for plugin in plugin_classes: 50 | LOGGER.debug(f"[🔧] Configuring account index ship: {plugin.__name__}") 51 | 52 | # Register the plugin: 53 | registry[plugin.__name__] = plugin # noqa 54 | LOGGER.debug(f"[🌟] Account Index: {plugin.__name__} has been discovered.") 55 | 56 | # Load the chosen one in the main configuration 57 | chosen_index_plugin = STARFLEET_CONFIGURATION.config["STARFLEET"]["AccountIndex"] 58 | index = registry.get(chosen_index_plugin) 59 | if not index: 60 | LOGGER.error( 61 | f"[🤷] Can't find the chosen account index plugin: {chosen_index_plugin}. The following {len(registry)} plugins were detected:" 62 | f" {','.join(registry.keys())}" 63 | ) 64 | raise UnknownAccountIndexError(chosen_index_plugin) 65 | 66 | # Instantiate the index: 67 | LOGGER.debug(f"[🏗️] Loading up the index: {chosen_index_plugin}...") 68 | self._index = index() 69 | 70 | except Exception as exc: 71 | LOGGER.error("[💥] Problem loading the account index plugins. See the stacktrace for details.") 72 | LOGGER.exception(exc) 73 | raise 74 | 75 | @property 76 | def index(self) -> AccountIndexInstance: 77 | """This will prepare the selected index for use.""" 78 | if not self._index: 79 | self.load_indexes() 80 | 81 | return self._index 82 | 83 | 84 | ACCOUNT_INDEX = StarfleetAccountIndexLoader() 85 | -------------------------------------------------------------------------------- /mkdocs/developerGuide/primaryComponents/workerShips/CLI.md: -------------------------------------------------------------------------------- 1 | # CLI - Command Line Interface Plugins 2 | 3 | We provide a CLI in Starfleet so that we are able to invoke the workers on demand locally. This assists with debugging and the occasional need to invoke a workload. 4 | 5 | To make expose CLIs, you need to make use of Click `Group`s. The `AccountIndexGeneratorShip` provides an example: 6 | 7 | ```python 8 | @click.group() 9 | def account_inventory() -> None: 10 | """This is the worker ship for generating an S3 account inventory""" 11 | 12 | 13 | @account_inventory.command() 14 | @click.option("--payload", required=True, type=click.File("r"), callback=load_payload, help="This is the worker payload YAML") 15 | @click.option("--commit", is_flag=True, default=False, show_default=True, help="Must be supplied for changes to be made") 16 | def generate(payload: Dict[str, Any], commit: bool) -> None: 17 | """This will generate an AWS account inventory from the organizations API""" 18 | if not commit: 19 | LOGGER.warning("[⚠️] Commit flag is disabled: not saving report to S3") 20 | 21 | worker = AccountIndexGeneratorShip() 22 | worker.load_template(payload) 23 | worker.execute(commit=commit) 24 | 25 | LOGGER.info("[✅] Done!") 26 | ``` 27 | 28 | !!! tip 29 | You will really want to get familiar with [Click's documentation](https://click.palletsprojects.com/) for groups and commands. 30 | 31 | 32 | ## The Click `Group` 33 | All CLI commands are to be a part of a `click` group. The groups are a collection of commands. The general gist for this is that when the CLI is used, it's going to be: `starfleet GROUP COMMAND ARGS`. 34 | 35 | We first make a function that is _named_ as the command group we want. In the example above, we wrap a function named `account_inventory` with a `@click.group()` decorator. This will make it so that `starfleet account-inventory` is a command available (`click` will substitute underscores (`_`) with hyphens (`-`) automatically for you). 36 | 37 | You can make as many groups as you want. We will discuss later how to make it so that these are loaded on start up. 38 | 39 | ## The Click `Command`s 40 | You will define the commands themselves with Click commands that are a part of the `Group`. This is done by making a function that is wrapped with the group function as a decorator like in the example: 41 | 42 | ```python 43 | @account_inventory.command() 44 | # ... 45 | def generate(payload: Dict[str, Any], commit: bool) -> None: 46 | # ... 47 | ``` 48 | 49 | Put together it's: 50 | 51 | ```python 52 | @click.group() 53 | def your_group() -> None: 54 | pass 55 | 56 | @your_group.command() 57 | def your_command() -> None: 58 | pass 59 | ``` 60 | In this example, the command to execute it is `starfleet your-group your-command`. You can add in as many commands as you want. 61 | 62 | ## Recommended Options 63 | We recommend that you always add a `--commit` flag to your commands. Please copy and paste: 64 | 65 | ```python 66 | @click.option("--commit", is_flag=True, default=False, show_default=True, help="Must be supplied for changes to be made") 67 | ``` 68 | 69 | ... and decorate your command with it. 70 | 71 | We also provide a convenience option for loading a payload YAML file to supply to your worker ship. To use it, copy and paste: 72 | 73 | ```python 74 | @click.option("--payload", required=True, type=click.File("r"), callback=load_payload, help="This is the worker payload YAML") 75 | ``` 76 | The main thing here is the `callback=load_payload`, which does a some work to load the payload. Note: this does not validate the payload; you will still need to do that. 77 | 78 | ### General Usage 79 | In general, you will want to just copy and paste what we have in the `AccountIndexGeneratorShip`, which for commands you will always want something that will instantiate the worker ship, load the payload template, and then execute the workload. 80 | 81 | See above for the example. 82 | -------------------------------------------------------------------------------- /mkdocs/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to ✨Starfleet✨ 2 | Welcome to the [Starfleet](https://github.com/gemini-oss/starfleet) project! Starfleet is a totally awesome whole-AWS infrastructure automation tool built off of AWS Lambda and Python. 3 | 4 | Starfleet Logo 5 | 6 | Starfleet is a tool that runs automation against an entire AWS infrastructure. The primary goal is to address gaps that exist with AWS Organizations. 7 | 8 | This works by running Lambda functions to operate in the context of an AWS account. Starfleet consists of worker "ships" that receive a templated payload 9 | to implement a desired infrastructure state. The "ship" will perform the actions necessary to ensure that the infrastructure state conforms to the template. 10 | Think of this like an IaC but for an entire AWS infrastructure. This allows you to place properly configured infrastructure resources where you need them. 11 | 12 | Starfleet's primary audience are cloud infrastructure and security engineers that have a need to support medium to large and growing AWS infrastructures. 13 | 14 | ## Why use Starfleet? 15 | We designed Starfleet to address the infrastructure needs of multi-account AWS users. In multi-account environments there is a need to configure cloud infrastructure that spans 16 | the entirety of the cloud footprint. In some cases, some accounts need to have differently configured resources than other accounts. Sometimes this is needed at a regional level too. 17 | 18 | Cloud infrastructure and security engineers need a place where they can easily design and implement features in a manner that is: 19 | 20 | 1. Easy to understand; human readable 21 | 1. Located in one convenient place 22 | 1. Flexible to the address the realities of medium to large and diverse AWS infrastructures 23 | 1. Infrastructure defined as code (IaC) 24 | 1. Detects AND prevents drift for incorrectly configured components 25 | 1. First-class understanding of multi-account and multi-regional workloads (or some combination thereof) 26 | 1. Dynamically and automatically operate across your AWS infrastructure as new accounts get provisioned and deleted 27 | 28 | A lot of existing tools come close for implementing some of these features, but don't quite meet the mark unless a lot of tooling is developed to address the shortcomings. 29 | 30 | ### Clear and Simple Templates 31 | Starfleet relies on simple, easy to read YAML templates that provide worker Lambda functions with the context required to operate across an AWS infrastructure. The design of these templates 32 | follows the following philosophy: 33 | 34 | 1. Control structures like loops and if statements belong in code, not templates 35 | 1. There is a time and place for Jinja; just not all of the times in all of the places 36 | 37 | The worker ships do all the heavy lifting so that templates are easy to read and parse. Easily readable templates provide context faster and reduce the likelihood of errors. 38 | This is especially important when operating at scale. We believe complexity should happen in the code; not N times in templates (the concept of _Don't Repeat Yourself_, only for real). 39 | 40 | At the end of the day, Starfleet is what you make of it. It provides the platform for running AWS account-aware Lambda functions that can operate anywhere in your infrastructure. 41 | 42 | ## Architecture Overview 43 | For more information please review the [Architecture](architecture/Overview.md) page for how this all works, but below is a sample diagram of the architecture: 44 | ![Starfleet Architecture](images/StarfleetArchitecture.svg) 45 | 46 | ## Video Overview 47 | Starfleet was presented at [fwd:cloudsec](https://fwdcloudsec.org/) 2023. The video [is here](https://youtu.be/5hyLfhTjtmE): 48 | 49 | 50 | -------------------------------------------------------------------------------- /mkdocs/developerGuide/primaryComponents/workerShips/LambdaEntrypoints.md: -------------------------------------------------------------------------------- 1 | # Lambda Entrypoints 2 | 3 | For the worker ship to do anything, it needs to define the entrypoint for AWS Lambda to invoke it. This is effectively just a function that AWS Lambda will call. 4 | 5 | We have defined some convenience functions that make this an easy(ier) process. For this, we will take the `AccountIndexGeneratorShip` as an example: 6 | 7 | 8 | ```python 9 | @worker_lambda(AccountIndexGeneratorShip) 10 | def lambda_handler(event: Dict[str, Any], context: object, worker: AccountIndexGeneratorShipInstance, commit: bool) -> None: # noqa pylint: disable=W0613 11 | """This is the Lambda entrypoint for the AccountIndexGeneratorShip event from the Starbase.""" 12 | for record in event["Records"]: 13 | # Load the payload: 14 | payload = json.loads(record["body"]) 15 | LOGGER.debug(f"[⚙️] Processing Payload: {payload}") 16 | worker.load_template(payload) 17 | 18 | # Process it! 19 | worker.execute(commit=commit) 20 | 21 | LOGGER.info("[🏁] Completed generating the account index.") 22 | ``` 23 | 24 | There are a bunch of things to unpack. First: AWS Lambda needs a `lambda_handler` function. This is what Lambda will call when it wants to invoke your Lambda function. For consistency we should always name our Lambda handler function `lambda_handler`. There is usually some boilerplate work associated with Lambda and also with Starfleet. It's hard to fully abstract this away, however we are able to make it easy to copy and paste :). 25 | 26 | ## The Decorator 27 | You'll notice in the code above that we make use of a decorator called `@worker_lambda`. You _must_ use this decorator for your Lambda handler function. What does this do? This is defined in the `starfleet.worker_ships.lambda_utils` package. This does a number of _very nice_ things for you: 28 | 29 | 1. This will automatically load the Starfleet configuration when your worker starts up, and verify that worker's configuration is configured properly 30 | 1. This instantiates the worker class 31 | 1. This handles the `commit` flag parsing (see note on that below) 32 | 1. This calls out to your Lambda handler function with the AWS Lambda provided event, `context` object, and your instantiated worker class. 33 | 34 | The code for it even has a nice example in the doc strong on how to use this (copying here for convenience): 35 | 36 | ```python 37 | YourWorkerShipInstance = TypeVar("YourWorkerShipInstance", bound=YourStarfleetWorkerShipClass) 38 | 39 | 40 | @worker_lambda(WorkerShipClass) 41 | def lambda_handler(event: Dict[str, Any], context: object, worker: YourWorkerShipInstance, commit: bool) -> None: 42 | for record in event["Records"]: 43 | payload = json.loads(record["body"]) 44 | 45 | # Validate the payload: (don't worry about the exception handling -- that is done in the decorator!) 46 | LOGGER.debug(f"[⚙️] Processing Payload: {payload}") 47 | worker.load_template(payload) 48 | 49 | # Process it! 50 | worker.execute(commit=commit) 51 | ``` 52 | 53 | An important note is that when you wrap your `lambda_handler` function with the decorator, you need to supply the class (not the instantiation!) into the decorator. I.e. if you have a worker ship class named `FooWorkerShip`, then your lambda handler will look like this: 54 | 55 | ```python 56 | FooWorkerShipInstance = TypeVar("FooWorkerShipInstance", bound=FooWorkerShip) 57 | 58 | @worker_lambda(FooWorkerShip) # <-- Pass in the worker ship class here 59 | def lambda_handler(event: Dict[str, Any], context: object, worker: FooWorkerShipInstance, commit: bool) -> None: 60 | # ... 61 | ``` 62 | 63 | Once you do that, you get all the benefits you need! 64 | 65 | ## Copy and Paste Time! 66 | Once you do have the decorated function correct, then you will just want to copy and paste in the `for` loop, as is. This will handle multiple events being provided by Lambda (you can also configure that in AWS SAM later), and it will verify and load the payload template, and then execute the workload. 67 | -------------------------------------------------------------------------------- /src/starfleet/utils/configuration.py: -------------------------------------------------------------------------------- 1 | """Starfleet's configuration loader and manager 2 | 3 | This makes use of YAML files in a very simple, but naive manner. This will simply load all YAML files into a dictionary that is used throughout the application. 4 | 5 | :Module: starfleet.utils.configuration 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import logging 12 | import os 13 | from typing import Any, Dict 14 | 15 | import yaml 16 | 17 | from starfleet.utils.config_schema import BaseConfigurationSchema 18 | from starfleet.utils.logging import LOGGER 19 | import starfleet 20 | 21 | CONFIGURATION_FILE_DIR_NAME = "configuration_files" 22 | PRE_LOGGER_LEVEL = os.environ.get("PRE_LOGGER_LEVEL", "DEBUG") 23 | 24 | 25 | class BadConfigurationError(Exception): 26 | """Exception for bad Starfleet configuration""" 27 | 28 | 29 | class StarfleetConfigurationLoader: 30 | """Class that loads Starfleet configuration files.""" 31 | 32 | # Defined here for testability purposes: 33 | _configuration_path = f"{starfleet.__path__[0]}/{CONFIGURATION_FILE_DIR_NAME}" 34 | 35 | def __init__(self): 36 | self._app_config: Dict[str, Any] = None # noqa 37 | 38 | def load_base_configuration(self) -> None: 39 | """This will load the base configuration for the application.""" 40 | self._app_config = {} 41 | 42 | # Set up the pre-logger, which is a logger that exists before we have a proper logger set up as we have not yet loaded a configuration! 43 | LOGGER.setLevel(PRE_LOGGER_LEVEL) 44 | LOGGER.debug(f"[📄] Loading the base configuration from {self._configuration_path}...") 45 | 46 | # This will load all files that end in .yaml from the starfleet/configuration_files/ path: 47 | try: 48 | for file in os.listdir(self._configuration_path): 49 | if file.endswith(".yaml"): 50 | LOGGER.debug(f"[⚙️] Processing configuration file: {file}...") 51 | 52 | with open(f"{self._configuration_path}/{file}", "r", encoding="utf-8") as stream: 53 | loaded = yaml.safe_load(stream) 54 | 55 | self._app_config.update(loaded) 56 | 57 | LOGGER.debug(f"[⚙️] Successfully loaded configuration file: {file}") 58 | 59 | except Exception as exc: 60 | LOGGER.error("[💥] Major error encountered loading configuration. Cannot proceed.") 61 | LOGGER.exception(exc) 62 | raise 63 | 64 | # Verify that the required components are in the configuration: 65 | try: 66 | errors = BaseConfigurationSchema().validate(self._app_config) 67 | if errors: 68 | raise BadConfigurationError(errors) 69 | 70 | except BadConfigurationError as bce: 71 | LOGGER.error("[💥] The Starfleet configuration is invalid. See the stacktrace for more details.") 72 | LOGGER.exception(bce) 73 | raise 74 | 75 | LOGGER.debug("[🪵] Configuring the logger for the rest of the application...") 76 | 77 | # Now, configure the logger from the loaded configuration: 78 | LOGGER.setLevel(self._app_config["STARFLEET"].get("LogLevel", PRE_LOGGER_LEVEL)) 79 | 80 | # Update the third_party_logger_levels if specified: 81 | for logger_name, level in self._app_config["STARFLEET"].get("ThirdPartyLoggerLevels", {}).items(): 82 | logging.getLogger(logger_name).setLevel(level) 83 | 84 | LOGGER.debug("[🆗️] Base configuration loaded successfully") 85 | 86 | @property 87 | def config(self) -> Dict[str, Any]: 88 | """Lazy-loads the application configuration. If not already loaded it will load the base configuration and then return it.""" 89 | if not self._app_config: 90 | self.load_base_configuration() 91 | 92 | return self._app_config 93 | 94 | 95 | STARFLEET_CONFIGURATION = StarfleetConfigurationLoader() 96 | -------------------------------------------------------------------------------- /tests/account_index/test_account_index_components.py: -------------------------------------------------------------------------------- 1 | """Tests for Stafleet's generic Account Index components 2 | 3 | Tests for the account index components, like the plugin loader 4 | 5 | :Module: starfleet.tests.account_index.test_account_index_components 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | # pylint: disable=unused-argument 12 | from typing import Any, Dict 13 | from unittest import mock 14 | 15 | import pytest 16 | 17 | from starfleet.account_index.loader import AccountIndexInstance 18 | 19 | 20 | def test_load_good_plugin(test_index: AccountIndexInstance) -> None: 21 | """This tests that the good testing plugin loads properly. This also tests the fixture.""" 22 | assert "000000000020" in test_index.get_all_accounts() 23 | 24 | 25 | def test_loading_invalid_subclass(test_configuration: Dict[str, Any]) -> None: 26 | """This tests that we properly handle exceptions related to incorrect subclasses.""" 27 | from starfleet.account_index.loader import StarfleetAccountIndexLoader 28 | from starfleet.utils.plugin_loader import InvalidPluginClassException 29 | import tests.account_index.testing_plugins.bad_plugin 30 | 31 | index_loader = StarfleetAccountIndexLoader() 32 | index_loader._index_ship_path = tests.account_index.testing_plugins.__path__ 33 | index_loader._index_ship_prefix = tests.account_index.testing_plugins.__name__ + "." 34 | 35 | # Test one that doesn't properly subclass the AccountIndex: 36 | class FakePlugin: 37 | """This is a fake plugin used for testing.""" 38 | 39 | fake_account_index_plugins = [FakePlugin] 40 | tests.account_index.testing_plugins.bad_plugin.ACCOUNT_INDEX_PLUGINS = fake_account_index_plugins 41 | with pytest.raises(InvalidPluginClassException) as exc: 42 | index_loader.load_indexes() 43 | assert str(exc.value).endswith("does not properly subclass: AccountIndex") 44 | 45 | del tests.account_index.testing_plugins.bad_plugin.ACCOUNT_INDEX_PLUGINS # noqa 46 | assert not hasattr(tests.account_index.testing_plugins.bad_plugin, "ACCOUNT_INDEX_PLUGINS") 47 | 48 | 49 | def test_loading_invalid_configuration(test_configuration: Dict[str, Any]) -> None: 50 | """This tests that the Starfleet configuration is set to load a plugin that exists.""" 51 | from starfleet.account_index.loader import StarfleetAccountIndexLoader, UnknownAccountIndexError 52 | import tests.account_index.testing_plugins 53 | 54 | test_configuration["STARFLEET"]["AccountIndex"] = "FakePlugin" 55 | index_loader = StarfleetAccountIndexLoader() 56 | index_loader._index_ship_path = tests.account_index.testing_plugins.__path__ 57 | index_loader._index_ship_prefix = tests.account_index.testing_plugins.__name__ + "." 58 | 59 | with pytest.raises(UnknownAccountIndexError) as exc: 60 | _ = index_loader.index 61 | 62 | assert str(exc.value) == "FakePlugin" 63 | 64 | 65 | def test_boto_outdated_logic(test_configuration: Dict[str, Any]) -> None: 66 | """This tests the logic for detecting updated boto3 regions.""" 67 | from starfleet.utils.niceties import get_all_regions 68 | from starfleet.account_index.loader import StarfleetAccountIndexLoader 69 | import tests.account_index.testing_plugins 70 | 71 | all_regions = get_all_regions() 72 | all_regions.add("some-new-region") 73 | 74 | with mock.patch("tests.account_index.testing_plugins.basic_plugin.get_all_regions", return_value=all_regions): 75 | with pytest.warns(UserWarning) as warning: 76 | account_indexer = StarfleetAccountIndexLoader() 77 | account_indexer._index_ship_path = tests.account_index.testing_plugins.__path__ 78 | account_indexer._index_ship_prefix = tests.account_index.testing_plugins.__name__ + "." 79 | index = account_indexer.index 80 | 81 | assert index.regions_map["some-new-region"] == index.account_ids 82 | 83 | assert "boto3 was updated" in str(warning.list[0].message) 84 | -------------------------------------------------------------------------------- /src/starfleet/account_index/schematics.py: -------------------------------------------------------------------------------- 1 | """Definitions for Account Index plugins. 2 | 3 | This defines what is required for making an Account Index. The Account Index is a special plugin that is used by Starfleet to know which accounts are where. 4 | This is what allows the Starbase to properly task Account and Account-Region payloads. 5 | 6 | :Module: starfleet.account_index.schematics 7 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 8 | :License: See the LICENSE file for details 9 | :Author: Mike Grima 10 | """ 11 | 12 | from typing import Set, Dict, Optional, TypeVar 13 | 14 | 15 | class AccountIndex: # pragma: no cover 16 | """ 17 | This is the base class that ALL account index plugins in Starfleet need to subclass. 18 | 19 | Make sure that you put your boostrapping code in the __init__ function. That is where you will want to do things like load the configuration, or load up the index. 20 | """ 21 | 22 | def get_accounts_by_ids(self, ids: Set[str]) -> Set[str]: 23 | """Return back a Set of account IDs for a given set of IDs present -- this effectively only returns back account IDs that exist in the inventory.""" 24 | raise NotImplementedError("Pew Pew Pew") 25 | 26 | def get_accounts_by_aliases(self, aliases: Set[str]) -> Set[str]: 27 | """Return back a Set of account IDs for a given set of aliases""" 28 | raise NotImplementedError("Pew Pew Pew") 29 | 30 | def get_accounts_by_tag(self, tag_name: str, tag_value: str) -> Set[str]: 31 | """Return back a set of account IDs based on the tag name and value pair""" 32 | raise NotImplementedError("Pew Pew Pew") 33 | 34 | def get_accounts_by_ou(self, org_unit: str) -> Set[str]: 35 | """Return back a set of account IDs based on the OU membership""" 36 | raise NotImplementedError("Pew Pew Pew") 37 | 38 | def get_accounts_by_regions(self, regions: Set[str]) -> Dict[str, Set[str]]: 39 | """Return back a dictionary of the region and the set of accounts associated with it.""" 40 | raise NotImplementedError("Pew Pew Pew") 41 | 42 | def get_accounts_for_all_regions(self) -> Dict[str, Set[str]]: 43 | """Return back a dictionary of the region and the set of all accounts associated with it -- but for ALL regions.""" 44 | raise NotImplementedError("Pew Pew Pew") 45 | 46 | def get_all_accounts(self) -> Set[str]: 47 | """Return back a set of all account IDs.""" 48 | raise NotImplementedError("Pew Pew Pew") 49 | 50 | def get_org_roots(self) -> Set[str]: 51 | """ 52 | Return back the set of account IDs for Organization Root accounts. This is mostly used for the Account and Account/Region worker ships when specifying if the payload 53 | in question should operate in an AWS Organization Root account. If the flag in the template is set, then this will check if there is an Organization Root set and task a 54 | worker ship with the payload to operate in that corresponding organization root. 55 | 56 | Note: The typical and preferred implementation for Starfleet is to operate on 1 AWS Organization, however, there is no reason why it can't operate over many. 57 | If you only have 1 org, just return the one account that is the org root account. 58 | """ 59 | raise NotImplementedError("Pew Pew Pew") 60 | 61 | def get_account_names(self, account_ids: Set[str]) -> Dict[str, Optional[str]]: 62 | """Return back a mapping of account ID to account name for the given set of account IDs. If the account ID is not found, it's mapped value is None.""" 63 | raise NotImplementedError("Pew Pew Pew") 64 | 65 | def get_account_tags(self, account_ids: Set[str]) -> Dict[str, Optional[Dict[str, str]]]: 66 | """Return back a mapping of account ID to the dictionary of name/value key pairs for the corresponding tags the account has (or doesn't have - empty dict)""" 67 | raise NotImplementedError("Pew Pew Pew") 68 | 69 | 70 | AccountIndexInstance = TypeVar("AccountIndexInstance", bound=AccountIndex) 71 | -------------------------------------------------------------------------------- /src/starfleet/configuration_files/configuration.yaml: -------------------------------------------------------------------------------- 1 | # The values in this file are based on the included AWS SAM template. 2 | # See https://gemini-oss.github.io/starfleet/installation/PrepareConfiguration/ for more details. 3 | 4 | # This is the main configuration stanza for Starfleet and the Starbase: 5 | STARFLEET: 6 | # REPLACE with the region for deployment 7 | DeploymentRegion: us-east-1 8 | 9 | # REPLACE with the Account ID for deployment 10 | TemplateBucket: starfleet-templates-DEPLOYMENT-ACCOUNT-ID 11 | 12 | # REPLACE with both the region for deployemnt and also the Account ID for deployment 13 | FanOutQueueUrl: https://sqs.DEPLOYMENT-REGION.amazonaws.com/DEPLOYMENT-ACCOUNT-ID/starbase-fanout-queue 14 | 15 | # Only replace if you have your own Account Index plugin defined: 16 | AccountIndex: StarfleetDefaultAccountIndex 17 | 18 | # Only use this if you have to store Secrets for your workers (example is the GitHubSyncWorker) 19 | # SecretsManager: 20 | # SecretId: starfleet 21 | # SecretRegion: us-east-2 22 | 23 | LogLevel: DEBUG 24 | ThirdPartyLoggerLevels: 25 | botocore: CRITICAL 26 | 'urllib3.connectionpool': CRITICAL 27 | 28 | 29 | # This is the main configuration stanza for the Account Index Generator Worker Ship 30 | # This is the worker that will go out and get the list of AWS accounts, and some details 31 | # about those accounts like the Parent OUs, Tags, and the enabled regions for those accounts 32 | # and store the results in a JSON file in the S3 Account Index Bucket 33 | AccountIndexGeneratorShip: 34 | Enabled: True 35 | 36 | # Only replace if you want your YAML to be called something else: 37 | TemplatePrefix: AccountIndexGenerator/SaveAccountInventory.yaml 38 | 39 | # REPLACE with both the region for deployemnt and also the Account ID for deployment 40 | InvocationQueueUrl: https://sqs.DEPLOYMENT-REGION.amazonaws.com/DEPLOYMENT-ACCOUNT-ID/starfleet-account-index-generator 41 | InvocationSources: 42 | - EVENTBRIDGE_TIMED_EVENT 43 | - S3 44 | EventBridgeTimedFrequency: HOURLY 45 | 46 | # This needs to be the name of the IAM role that is created in the Installation documentation for StackSets 47 | # (but requires you to create it manually because StackSets doesn't operate in the org root, which is one 48 | # of the many reasons we are making Starfleet...) 49 | OrgAccountAssumeRole: starfleet-worker-basic-test-role # This is test deployment role. Swap with `prod` for prod. 50 | 51 | # REPLACE with the AWS account ID for the AWS account that is the organization root: 52 | OrgAccountId: "REPLACEME" 53 | 54 | # REPLACE with the actual AWS Organization root ID (this is different than the account ID). 55 | # You can find this in the AWS Organizations UI 56 | OrgRootId: "r-eplaceme" 57 | 58 | # This needs to be the name of the IAM role that is created in the Installation documentation for StackSets 59 | # (but requires you to create it manually because StackSets doesn't operate in the org root, which is one 60 | # of the many reasons we are making Starfleet...) 61 | DescribeRegionsAssumeRole: starfleet-worker-basic-test-role # This is test deployment role. Swap with `prod` for prod. 62 | 63 | 64 | # This is the configuration stanza for the default Account Index plugin. This uses the account index generated by 65 | # the AccountIndexGeneratorShip to build out an account index that the Starbase can use to task the worker ships 66 | # NOTE: You will first need to have the inventory generated in S3 before the Starbase can run. You can do this by 67 | # running the the CLI for the AccountIndexGeneratorShip. 68 | StarfleetDefaultAccountIndex: 69 | 70 | # REPLACE with the Account ID for deployment 71 | IndexBucket: starfleet-account-index-DEPLOYMENT-ACCOUNT-ID 72 | 73 | # REPLACE with the AWS Region for where the bucket above resides. The SAM template puts this in the same region 74 | # as the rest of Starfleet's resources: 75 | BucketRegion: REPLACE-ME 76 | 77 | # The default is accountIndex.json -- no need to uncomment unless you want to specify a 78 | # different path. 79 | # IndexObjectPath: accountIndex.json 80 | -------------------------------------------------------------------------------- /tests/account_index/testing_plugins/basic_plugin/__init__.py: -------------------------------------------------------------------------------- 1 | """A sample account index plugin used for unit testing 2 | 3 | Sample account index plugin for unit testing purposes. 4 | 5 | :Module: starfleet.tests.account_index.testing_plugins.basic_plugin 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import json 12 | import warnings 13 | from typing import Dict, Set 14 | 15 | from starfleet.account_index.plugins.starfleet_default_index import StarfleetDefaultAccountIndex 16 | from starfleet.utils.niceties import get_all_regions 17 | 18 | import tests.starfleet_included_plugins.account_index_generator 19 | 20 | 21 | class TestingAccountIndexPlugin(StarfleetDefaultAccountIndex): 22 | """This is mostly a clone of the StarfleetDefaultAccountIndex, but slimmed down specifically for unit testing the Starbase.""" 23 | 24 | # Intentionally not calling the super __init__ in this function (to not reach out to S3): 25 | def __init__(self): # noqa pylint: disable=super-init-not-called 26 | """This will load the generatedIndex.json file that is used by the tests.starfleet_included_plugins.account_index_generator tests.""" 27 | self.account_ids = set() 28 | self.alias_map: Dict[str, str] = {} 29 | self.account_name_map: Dict[str, str] = {} 30 | self.ou_map: Dict[str, Set[str]] = {} 31 | self.regions_map: Dict[str, Set[str]] = {} 32 | self.tag_map: Dict[str, Dict[str, Set[str]]] = {} # Dict of tag name -> tag value -> accounts 33 | self.account_tag_map: Dict[str, Dict[str, str]] = {} # Dict of account ID -> tag dictionary 34 | 35 | path = f"{tests.starfleet_included_plugins.account_index_generator.__path__[0]}/generatedIndex.json" 36 | with open(path, "r", encoding="utf-8") as file: 37 | account_dict = json.loads(file.read())["accounts"] 38 | 39 | # NOTE: Boto updates will change the supported regions for accounts. If we detect that the regions in the loaded index file are not the same as what boto3 now supports, 40 | # then we will override what is in the JSON with whatever the latest regions supported by boto3 to prevent tests from breaking 41 | # However, we will issue a warning if we detect this as it's not ideal to have the loaded tests not use the raw values from the file. 42 | all_regions = sorted(get_all_regions()) 43 | append_regions = [] 44 | 45 | for region in all_regions: 46 | if region not in account_dict["000000000001"]["Regions"]: 47 | append_regions.append(region) 48 | 49 | # If we have found missing regions, then boto3 was updated with new regions that are not present in the account index. 50 | if append_regions: 51 | corrected_regions = sorted(account_dict["000000000001"]["Regions"] + append_regions) 52 | 53 | warnings.warn( 54 | "[⚠️] Note: boto3 was updated and the generated index: `tests/starfleet_included_plugins/account_index_generator/generatedIndex.json` has an outdated list of " 55 | "regions. Please update the generatedIndex.json file by following the instructions on this file (the one that emitted this warning)." 56 | ) 57 | 58 | # Instructions: 59 | # To update the generatedIndex.json file, you need to go to: src/starfleet/worker_ships/plugins/account_index_generator/ship.py, 60 | # and at the bottom of the `AccountIndexGeneratorShip` class, there is code you need to uncomment (these instructions are there as well). You have to run the unit tests 61 | # for the account index generator worker with those lines of code uncommented, and then clone over 62 | # that file to 'tests/starfleet_included_plugins/account_index_generator/generatedIndex.json'. Once finished, remember to re-comment out those lines. 63 | 64 | for account in account_dict.values(): 65 | account["Regions"] = corrected_regions 66 | 67 | self._load_inventory(account_dict) 68 | 69 | 70 | ACCOUNT_INDEX_PLUGINS = [TestingAccountIndexPlugin] 71 | -------------------------------------------------------------------------------- /mkdocs/developerGuide/primaryComponents/workerShips/Loader.md: -------------------------------------------------------------------------------- 1 | # Worker Ship and CLI Loaders 2 | 3 | This section covers how we make it all work! At this point you would have set up a worker ship plugin complete with all the configuration and payload schemas. Great! 🎉 But, Starfleet won't yet know about your ship yet. This is where the worker ship loader comes in. 4 | 5 | ## Make Starfleet See Your Worker and CLIs 6 | In your worker ship Python package, you will have an `__init__.py` file. In that file you _**must**_ to define the following: 7 | 8 | ```python 9 | WORKER_SHIP_PLUGINS = [YOUR, PLUGIN, CLASSES, HERE] 10 | ``` 11 | 12 | If you want to expose CLIs, then in that same `__init__`.py file, file you _**must**_ to define the following: 13 | 14 | ```python 15 | CLICK_CLI_GROUPS = [YOUR, CLICK, GROUPS, HERE] 16 | ``` 17 | 18 | A great example is the `AccountIndexGeneratorShip`'s `__init__.py`, which looks like this: 19 | 20 | ```python 21 | from starfleet.worker_ships.plugins.account_index_generator.ship import AccountIndexGeneratorShip, account_inventory 22 | 23 | WORKER_SHIP_PLUGINS = [AccountIndexGeneratorShip] # NEED TO DEFINE THIS FOR STARFLEET TO KNOW ABOUT YOUR PLUGIN! 24 | CLICK_CLI_GROUPS = [account_inventory] # NEED TO DEFINE THIS FOR STARFLEET TO KNOW ABOUT YOUR CLI! 25 | ``` 26 | 27 | The `WORKER_SHIP_PLUGINS` is a list of the worker ship plugin classes that you want Starfleet to see. You can add in as many plugins as you want. Starfleet will locate this list and then register the worker ships to the worker ship loader so that is can be interacted with. This is what will allow the Starbase to actually find your worker ship plugin, and task it. 28 | 29 | The `CLICK_CLI_GROUPS` works the exact same way! Only for that, you are adding in the list of Click `Group` functions. On startup, there is a CLI loader that will pick this up and register all the commands associated with it dynamically. 30 | 31 | ## The Worker Ship Loader 32 | You don't need to know the full context of this, but it's here should you be interested in knowing. 33 | 34 | Starfleet has a singleton defined in `starfleet.worker_ships.loader` called `STARFLEET_WORKER_SHIPS`. This is a class that will lazy load all the worker ships and verify that worker ship is configured properly and enabled. It does this by iterating through the Python packages that reside in `starfleet.worker_ships.plugins` 35 | 36 | Calling the `STARFLEET_WORKER_SHIPS.get_worker_ships()` method returns a dictionary of the worker ship name and the enabled and instantiated worker ship object for interaction. 37 | 38 | Feel free to dig around the code in `starfleet.worker_ships.loader` for more details on what that's doing. During unit testing we have a pytest fixture mock out the location of where the worker ships reside to `tests.worker_ship_utils.testing_plugins`. Take a look at the code in those locations for other examples of very basic Starfleet worker ship plugins. 39 | 40 | ## The CLI Loader 41 | You also don't need to know the full context of this, but it's provided here for context. 42 | 43 | This is super similar in concept to the Worker Ship Loader, but is used to register all the Click commands. Starfleet leverages [Python entrypoints](https://packaging.python.org/en/latest/specifications/entry-points/) in combination with [Click's support for it](https://click.palletsprojects.com/en/8.1.x/setuptools/). This is defined in `pyproject.toml` with: 44 | 45 | ```toml 46 | [project.scripts] 47 | starfleet = "starfleet.cli.entrypoint:cli" 48 | ``` 49 | 50 | The CLI loader has a class defined in `starfleet.cli.entrypoint`. This is the main Click group that is used. This is the `starfleet ...` command. This command group uses a custom class to load up all the CLIs named `StarfleetClickGroup`. `StarfleetClickGroup` is a class that sub-classes `click.Group` (defined in `starfleet.cli.components`), and it will, on startup: 51 | 52 | 1. Output our awesome Starfleet text logo 53 | 1. Set up the Starfleet configuration, load it, and verify it's all good 54 | 1. Load up all the worker ships, and verify they are all good 55 | 1. And then register all the CLI groups and commands to Click 56 | 57 | At this point Click will then do the logic required to run your command. All of this is done very quickly and seamlessly for you! -------------------------------------------------------------------------------- /tests/test_starfleet.py: -------------------------------------------------------------------------------- 1 | """Base tests for Starfleet. 2 | 3 | :Module: starfleet.tests.test_starfleet 4 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 5 | :License: See the LICENSE file for details 6 | :Author: Mike Grima 7 | """ 8 | 9 | import pytest 10 | 11 | 12 | def test_app_startup() -> None: 13 | """This tests that the application start up is able to load the utils properly.""" 14 | from starfleet.startup import STARFLEET_CONFIGURATION, base_start_up 15 | 16 | base_start_up() 17 | 18 | # Test that we have a proper utils loaded (not a super sophisticated test): 19 | assert STARFLEET_CONFIGURATION.config["STARFLEET"] 20 | 21 | 22 | def test_find_plugins() -> None: 23 | """This tests the find_plugins loading capability.""" 24 | import tests.worker_ship_utils.testing_plugins 25 | from tests.worker_ship_utils.testing_plugins.basic_plugin import WORKER_SHIP_PLUGINS 26 | from starfleet.utils.plugin_loader import find_plugins 27 | from starfleet.worker_ships.ship_schematics import StarfleetWorkerShip 28 | 29 | plugins = find_plugins( 30 | tests.worker_ship_utils.testing_plugins.__path__, # noqa 31 | tests.worker_ship_utils.testing_plugins.__name__ + ".", 32 | "WORKER_SHIP_PLUGINS", 33 | StarfleetWorkerShip, 34 | ) 35 | 36 | # Iterate over the module components and pull out the StarfleetWorkerShip objects: 37 | assert len(plugins) == 1 38 | assert plugins["tests.worker_ship_utils.testing_plugins.basic_plugin"] == WORKER_SHIP_PLUGINS 39 | assert issubclass(plugins["tests.worker_ship_utils.testing_plugins.basic_plugin"][0], StarfleetWorkerShip) 40 | 41 | 42 | def test_find_plugin_exceptions() -> None: 43 | """This tests that exceptions are properly raised in the find_plugins function. 44 | 45 | Possible TODO: If this causes errors, it may likely be because of concurrent test execution. If that happens, then make a separate package just for the 46 | bad plugin and pass in that path to not interfere with `starfleet.tests.worker_ship_utils.testing_plugins`. If not, then just leave it as is ;) 47 | """ 48 | import tests.worker_ship_utils.testing_plugins 49 | import tests.worker_ship_utils.testing_plugins.bad_plugin 50 | from starfleet.utils.plugin_loader import find_plugins, InvalidPluginClassException, InvalidPluginListException 51 | from starfleet.worker_ships.ship_schematics import StarfleetWorkerShip 52 | 53 | # First test that if we don't have a list in for the WORKER_SHIP_NAME (the plugin_attr_name) that this will fail (this is adding the field, so can't use mock): 54 | tests.worker_ship_utils.testing_plugins.bad_plugin.WORKER_SHIP_PLUGINS = "pew pew pew" 55 | with pytest.raises(InvalidPluginListException) as exc: 56 | find_plugins( 57 | tests.worker_ship_utils.testing_plugins.__path__, # noqa 58 | tests.worker_ship_utils.testing_plugins.__name__ + ".", 59 | "WORKER_SHIP_PLUGINS", 60 | StarfleetWorkerShip, 61 | ) 62 | assert ( 63 | str(exc.value) == "[💥] The package: tests.worker_ship_utils.testing_plugins.bad_plugin needs a variable named WORKER_SHIP_PLUGINS that is of type " 64 | "List, not type: " 65 | ) 66 | 67 | # Next, we are going to test that if we have a class name in WORKER_SHIP_NAME that is not based on StarfleetWorkerShip, that it will fail: 68 | tests.worker_ship_utils.testing_plugins.bad_plugin.WORKER_SHIP_PLUGINS = [str] 69 | with pytest.raises(InvalidPluginClassException) as exc: 70 | find_plugins( 71 | tests.worker_ship_utils.testing_plugins.__path__, # noqa 72 | tests.worker_ship_utils.testing_plugins.__name__ + ".", 73 | "WORKER_SHIP_PLUGINS", 74 | StarfleetWorkerShip, 75 | ) 76 | assert ( 77 | str(exc.value) == "[💥] The plugin: str in package: tests.worker_ship_utils.testing_plugins.bad_plugin does not properly subclass: StarfleetWorkerShip" 78 | ) 79 | 80 | # Clean Up just to be on the safe side for future tests: 81 | del tests.worker_ship_utils.testing_plugins.bad_plugin.WORKER_SHIP_PLUGINS # noqa 82 | assert not hasattr(tests.worker_ship_utils.testing_plugins.bad_plugin, "WORKER_SHIP_PLUGINS") 83 | -------------------------------------------------------------------------------- /mkdocs/userGuide/Secrets.md: -------------------------------------------------------------------------------- 1 | # Secrets Management in Starfleet 2 | 3 | Starfleet relies on an optional AWS Secrets Manager component for storing sensitive data, like tokens, passwords, and encryption keys. If you don't have a need for this then you don't need to make use of this. 4 | 5 | The [Developer Guide](../developerGuide/primaryComponents/SecretsManager.md) has the raw details on how this works programmatically, but for this guide, we are going to explore how to configure it and make use if it. 6 | 7 | !!! note "Cached Data Notice" 8 | Like the other components in Starfleet, the loaded data will persist in memory on subsequent runs of the Lambda function. This has the benefit of not needing to make repeated AWS Secrets Manager API calls, which saves money. The downside to this is that AWS will persist the secrets value in memory as long as the container running your Lambda function remains operational (this happens on the AWS backend; you don't have control over it). If you make an update to the secret string, subsequent Lambda calls may not yet see it and you could operate off of the old cached secrets value. 9 | 10 | AWS will periodically rotate the Lambda container out every few hours if no code has changed. The only way for you to force your Lambda function's container to rotate out is to actually update the code for your Lambda function. 11 | 12 | !!! tip "Slack" 13 | If you make use of Slack for alerts, then you need to configure the Secrets Management in Starfleet. Continue reading this page for details. 14 | 15 | ## The Secret 16 | Starfleet uses an AWS Secrets Manager secret for storing sensitive data. The data in the secret is stored as a JSON string that looks like this: 17 | 18 | ```json 19 | { 20 | "STARFLEET": { 21 | "SlackToken": "the-token-here" 22 | }, 23 | "WORKER_NAME": "Any value - this can be a string, nested Dict, etc.", 24 | "...More Workers Here..." 25 | } 26 | ``` 27 | 28 | You will need to create a secret named `Starfleet` in AWS secrets manager in the same account and region as Starfleet. The initial value for this JSON should look like this: 29 | ```json 30 | { 31 | "STARFLEET": {} 32 | } 33 | ``` 34 | 35 | Once this is saved, you will need to ensure that all Starfleet components that need access to this secret have their AWS Lambda function IAM roles be given the following permissions to access the secret: 36 | ```json 37 | { 38 | "Statement": [ 39 | { 40 | "Effect": "Allow", 41 | "Action": "secretsmanager:GetSecretValue", 42 | "Resource": "arn:aws:secretsmanager:REGION:ACCOUNT:secret:SECRET-ID-HERE" 43 | } 44 | ] 45 | } 46 | ``` 47 | 48 | We recommend making this a managed policy that is attached to all the worker IAM roles. 49 | 50 | ## Configuration 51 | If you are making use of the Secrets Management feature, you have to make sure that the Starfleet configuration is configured to point to it. Under the main `STARFLEET` stanza of the configuration you need to have the following: 52 | 53 | * **`SecretsManager`** - This is a dictionary that outlines the ID of an AWS Secrets Manager secret. This allows Starfleet workers to reference secrets. If specified, this contains 2 required fields: `SecretId` and `SecretRegion`. The ID is the name of the secret and the region is the region for it. The secret should reside in the same AWS account as Starfleet. 54 | 55 | This is what that looks like: 56 | 57 | ```yaml 58 | STARFLEET: 59 | # ... All the other stuff here ... 60 | SecretsManager: 61 | SecretId: Starfleet # This is the name of the AWS Secrets Manager secret. The secret must reside in the same AWS account as Starfleet. 62 | SecretRegion: us-east-2 # This is the AWS region for where the secret resides. 63 | 64 | WorkerName: 65 | Enabled: True 66 | # ... All the other stuff here ... 67 | ``` 68 | 69 | ## Example Secret Entry 70 | An example of a secret JSON string for a Starfleet setup with Slack and a GitHub application for the `GitHubSyncWorkerShip` would look something like this: 71 | 72 | ```json 73 | { 74 | "STARFLEET": { 75 | "SlackToken": "xoxb-REDACTED" 76 | }, 77 | "GitHubSyncWorker": { 78 | "GITHUB_ORG_NAME_HERE": "-----BEGIN RSA PRIVATE KEY-----\nREDACTED\nREDACTED\nREDACTED\n-----END RSA PRIVATE KEY-----" 79 | } 80 | } 81 | ``` 82 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/lambda_utils.py: -------------------------------------------------------------------------------- 1 | """Lambda utility functions for the worker ships. 2 | 3 | This module contains a number of utility functions for worker ships, like decorators. 4 | 5 | :Module: starfleet.worker_ships.lambda_utils 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | import os 12 | from functools import wraps 13 | from typing import Callable, Dict, Type, Any 14 | 15 | from starfleet.utils.configuration import BadConfigurationError, STARFLEET_CONFIGURATION 16 | from starfleet.utils.logging import LOGGER 17 | from starfleet.worker_ships.ship_schematics import StarfleetWorkerShip, AlertPriority 18 | 19 | # Set up the configuration now so loggers are properly configured: 20 | STARFLEET_CONFIGURATION.config # noqa pylint: disable=W0104 21 | 22 | 23 | def worker_lambda(worker_ship: Type[StarfleetWorkerShip]) -> Callable: 24 | """This is a decorator that does some convenience for the worker lambda handlers. You should always use this.""" 25 | 26 | def wrapped_worker_lambda(func: Callable) -> Callable: 27 | """Because this decorator takes in an argument, you need to return the real decorator.""" 28 | 29 | @wraps(func) 30 | def wrapped_lambda_handler(event: Dict[str, Any], context: object) -> None: # noqa 31 | """This is the wrapped function that will inject the instantiated worker with the configuration all validated. This also handles the commit flag handling. 32 | 33 | This takes in all the standard lambda handler items. 34 | 35 | Example usage: 36 | 37 | YourWorkerShipInstance = TypeVar("YourWorkerShipInstance", bound=YourStarfleetWorkerShipClass) 38 | 39 | 40 | @worker_lambda(WorkerShipClass) 41 | def lambda_handler(event: Dict[str, Any], context: object, worker: YourWorkerShipInstance, commit: bool) -> None: 42 | for record in event["Records"]: 43 | payload = json.loads(record["body"]) 44 | 45 | # Validate the payload: (don't worry about the exception handling -- that is done in the decorator!) 46 | LOGGER.debug(f"[⚙️] Processing Payload: {payload}") 47 | worker.load_template(payload) 48 | 49 | # Process it! 50 | worker.execute(commit=commit) 51 | """ 52 | worker = worker_ship() 53 | LOGGER.info(f"[🛸] Starting the {worker.worker_ship_name} Worker Ship...") 54 | 55 | # Get the configuration: 56 | worker_config = STARFLEET_CONFIGURATION.config.get(worker.worker_ship_name, {}) 57 | if not worker_config: 58 | LOGGER.error(f"[💥] Can't find configuration entry for: {worker.worker_ship_name}!") 59 | raise BadConfigurationError("[💥] No configuration found for the worker") 60 | 61 | errors = worker.configuration_template_class().validate(worker_config) 62 | if errors: 63 | LOGGER.error("[💥] Invalid configuration for the worker! See the stacktrace for details.") 64 | raise BadConfigurationError(f"[💥] Worker ship: {worker.worker_ship_name} has an invalid configuration. {str(errors)}") # noqa 65 | 66 | # If any combination of the string "true" is present, then it's True. The default is thus False. 67 | commit = os.environ.get("STARFLEET_COMMIT", "").lower() == "true" 68 | 69 | # Finally, inject the alert details if present in the configuration: 70 | alert_config = worker_config.get("AlertConfiguration", {"ChannelId": None, "AlertPriority": "NONE"}) 71 | priorities = {priority.name: priority for priority in AlertPriority} 72 | worker.alert_channel = alert_config["ChannelId"] 73 | worker.alert_priority = priorities[alert_config["AlertPriority"]] 74 | 75 | try: 76 | func(event, context, worker, commit) 77 | except Exception: 78 | LOGGER.error("[💥] Encountered major problem processing the payload. See the stacktrace for details!") 79 | raise 80 | 81 | return wrapped_lambda_handler 82 | 83 | return wrapped_worker_lambda 84 | -------------------------------------------------------------------------------- /mkdocs/developerGuide/primaryComponents/ConfigurationManager.md: -------------------------------------------------------------------------------- 1 | # Configuration Manager 2 | 3 | It is very important to understand the configuration! Make sure you first review the [Architecture section for Configuration](../../architecture/Configuration.md) before reading this section. The configuration is provided as a singleton object in `starfleet.utils.configuration` and is used like this: 4 | 5 | ```python 6 | # Import the configuration manager: 7 | from starfleet.utils.configuration import STARFLEET_CONFIGURATION 8 | 9 | # ... 10 | 11 | # Use the configuration 12 | some_config_entry = STARFLEET_CONFIGURATION.config["STARFLEET"] # Fetch the dictionary under the STARFLEET configuration section 13 | some_other_config_entry = STARFLEET_CONFIGURATION.config["AccountIndexGeneratorShip"] # Fetch the dictionary under the AccountIndexGeneratorShip worker ship definition 14 | ``` 15 | 16 | All of the configuration is obtained by using `STARFLEET_CONFIGURATION.config`. `config` is a dictionary that is lazy loaded property that contains a dictionary of all the configuration YAML data. The configuration is loaded once on startup by it going to the `src/starfleet/configuration_files/` path, loading all the nested `.yaml` files and then merging them into one large dictionary. 17 | 18 | ### Configuration Schemas 19 | There are Marshmallow schemas for configuration, however these are loaded piecemeal and not all at once. This can be confusing so we'll tackle this in parts to make this more clear. 20 | 21 | #### Base Configuration Schema 22 | There is a `BaseConfigurationSchema` that resides in `starfleet.utils.config_schema`, which defines the schema for the `STARFLEET` section of the configuration. This schema is used when loading the initial configuration, and verifies that the `STARFLEET` section is correct. 23 | 24 | #### Worker Ship Configurations 25 | The worker ship configurations are documented later in detail. However, for now we'll mention that the worker ship configurations are validated against the worker ship defined schemas when the worker ship plugins are loaded later on during startup. 26 | 27 | ### Unit Testing Configuration 28 | Mocking out the configuration is extremely important for running unit tests. We have defined a pytest fixture that sets the configuration manager's `config` property to a testing dictionary suitable for all the unit tests. The really nice thing about this is that this sets the testing configuration for *all* the unit tests as long as the `test_configuration` fixture is used during the tests (or inherited from another fixture you are using). This is another reason why we really like using Singletons - it makes mocking things out very easy and globally for the code when testing. 29 | 30 | The `test_configuration` fixture is defined in `tests.conftest`. Unit tests have configuration YAML files stored in a separate location under `tests/test_configuration_files/`. The configuration manager is configured by the fixture to load files from that location instead of `src/starfleet/configuration_files` to make testing clean and isolated. This is a pattern that is frequently used throughout Starfleet. 31 | 32 | As you develop features, you will want to make changes to the configuration to include the details you need for testing the code you are writing. You can easily do that by making a pytest fixture that looks like this: 33 | 34 | ```python 35 | @pytest.fixture 36 | def my_worker_ship_configuration(test_configuration: Dict[str, Any]) -> None: 37 | """This will inject my code's configuration into the configuration manager for use throughout the app.""" 38 | my_apps_config = {"some_field": "some_value", "some_other_field": "some_other_value"} 39 | 40 | test_configuration["MyWorkerShip"] = my_apps_config 41 | 42 | # ... 43 | 44 | def test_my_ships_configuration(my_worker_ship_configuration: None) -> None: 45 | """This tests that the configuration manager has the configuration set by my fixture loaded within it.""" 46 | from starfleet.utils.configuration import STARFLEET_CONFIGURATION 47 | 48 | assert STARFLEET_CONFIGURATION.config["MyWorkerShip"] == {"some_field": "some_value", "some_other_field": "some_other_value"} 49 | # ^^ This will be True 50 | ``` 51 | 52 | !!! tip 53 | You will want to really understand how pytest fixtures work. We use them extensively. Please review the pytest documentation for more details. 54 | -------------------------------------------------------------------------------- /mkdocs/installation/SetupECR.md: -------------------------------------------------------------------------------- 1 | # Set Up Private ECR (Elastic Container Registry) 2 | 3 | This is not required but highly recommended as the Starfleet package may be too big for Lambda to accept it as a `.zip` file. As such, we recommend setting up Starfleet with a Dockerized Lambda hosted in a private ECR. 4 | 5 | The steps to do this will largely involve setting up ECR, which you can find in [the AWS documentation here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html). 6 | 7 | When creating your registry please adhere to the following: 8 | 9 | 1. You are creating a **PRIVATE** registry. **DO NOT MAKE THIS PUBLIC!** 10 | 2. This must reside in the same account and region that you are deploying Starfleet to 11 | 3. Recommend that you set up tag immutability 12 | 4. Recommend that you set up a lifecycle policy to expire older images (recommend to expire any `Image count more than 3`) 13 | 14 | !!! danger "Public Access" 15 | DO NOT make your ECR repository public! 16 | 17 | Once you have your ECR registry set up, there will be a URL associated with it. That URL is the where your image will be pushed to. You will need that in the next steps for AWS SAM. 18 | 19 | !!! note "Note: ECR URL" 20 | You will need the ECR URL for the next sections. This is in the format of `aws_account_id.dkr.ecr.region.amazonaws.com` 21 | 22 | ## Note about Dockerizing Starfleet 23 | Included in the main code base is a `Dockerfile` that is intended to build Starfleet. This file should be all good to go. This is configured to use the AWS official Lambda docker container for a specific Python version. This also: 24 | 25 | 1. Copies the Starfleet code to the container in the `var/runtime` directory, which is where Lambda wants the code to live. 26 | 2. Cleans up some unnecessary files. 27 | 2. It runs `pip install` to install the Starfleet packages 28 | 3. Lastly, and _very importantly_ it **removes** the built-in `boto3` and `botocore` packages that is shipped by default with the container. This is necessary because Lambda will prefer to use the preloaded version of boto instead of the Starfleet packaged one. The pre-loaded version can be out of date and can result in very bizarre errors with unsupported boto APIs - this is because you may be using boto features that are newer than the included boto packages. Thus, we remove them so only the Starfleet included boto packages are there. 29 | 30 | For more information about Dockerizing Lambda functions, [see the docs here](https://docs.aws.amazon.com/lambda/latest/dg/python-image.html). 31 | 32 | ## Continuing 33 | 34 | At this point you will have the following: 35 | 36 | - [x] Enable AWS Organizations if you haven't already done so and move some accounts into it 37 | - [x] Pick out an AWS account for deploying a testing version of Starfleet 38 | - [x] Work on getting a read-only Starfleet IAM role deployed with the permissions outlined above in all your AWS accounts. This role is _not_ very permissive and is only able to describe the enabled regions for an account. 39 | - [x] In the organization root, it has permissions to list the Organizations accounts. 40 | - [x] If you use StackSets then you need to manually make the role in the org root since StackSets won't operate in the org root. 41 | - [x] Important: Make sure that you have some IAM principal that you can use locally that can assume all these roles. This will be needed to run the Starfleet CLI. If you use AWS SSO, then use the ARN for the permissions set provisioned administrative role in the Starfleet account. See the note above for an example. 42 | - [x] AWS Account identified for deployment 43 | - [x] Starfleet worker IAM roles deployed everywhere 44 | - [x] The `configuration.yaml` file in `src/starfleet/configuration_files` modified with values unique to your environment 45 | - [x] A payload template (not stored as a configuration file) in a different place than your configuration that describes what the Starfleet Account Index Generator is supposed to do 46 | - [x] An optional ECR Repository set up to make dockerized Lambda builds 47 | - [x] And Now: AWS SAM: 48 | - [x] SAM's administrative resources deployed 49 | - [x] SAM's TEST deployment configuration all set up 50 | - [x] Starfleet deployed in your environment 51 | 52 | While it's now deployed, it won't work without an Account Index. The next section describes how to get it set up. 53 | -------------------------------------------------------------------------------- /mkdocs/developerGuide/primaryComponents/AccountIndexer.md: -------------------------------------------------------------------------------- 1 | # Account Indexer 2 | 3 | The Account Index plugin is loaded by the Starbase and optionally other worker ships if they need it. [As mentioned](../../architecture/AccountIndex.md), the purpose is to provide an inventory of AWS accounts that Starfleet can operate on. 4 | 5 | An Account Indexer is required for the Starbase to function, and for your account and account/region based workers to function. Starfleet requires the name of an account index plugin to be provided in the `STARFLEET` stanza of the configuration with the field `AccountIndexer: AccountIndexPluginNameHere`. 6 | 7 | ## Default Account Index 8 | By default, Starfleet will leverage the `StarfleetDefaultAccountIndex` plugin. This plugin leverages the generated account index JSON that is produced by the `AccountIndexGeneratorShip` plugin saved to S3. You will first need to have that index generator run before any other workers can be used with this account index. 9 | 10 | If the default account index is not sufficient for your use case, then you are welcome to make your own. See below. 11 | 12 | ## Account Index Plugin Residency 13 | Account index plugins need to reside in the `src/starfleet/account_index/plugins/` directory. This is exactly the same pattern that is used for worker ship plugins. 14 | 15 | At a minimum, you'll need a `__init__.py` file. We'll cover more about this file in the Worker Ship Loader portion. For now, just now that you will need a directory that looks like this: 16 | 17 | ``` 18 | ... 19 | account_index 20 | └── plugins 21 | └── your_plugin 22 | └── __init__.py 23 | └── some_other_python_file.py 24 | └── ... 25 | ``` 26 | 27 | See the [Developer Guide Overview](../Overview.md#packaging-deployment-considerations) page on more details on packaging non-OSS and internal worker ship plugins. 28 | 29 | ## Base Class 30 | Like with worker ships, there is a base class named `AccountIndex` that resides in `starfleet.account_index.schematics` that you need to sub-class. This class has a number of methods that resolve AWS account IDs. The plugin needs to interact with whatever AWS account inventory system you use to pull out the requested details. Generally, the methods will return a Python set of AWS Account ID strings back out. The Starbase will heavily rely on Python `set` logic for coming up with the proper list of accounts to iterate over. 31 | 32 | ## Registering the Plugin 33 | Similar to the worker ships, the account index plugin needs to have a `__init__.py` file that has a list named `ACCOUNT_INDEX_PLUGINS` equal to a list of `AccountIndex` sub-classes. 34 | 35 | The `StarfleetDefaultAccountIndex` is a great example: 36 | ```python 37 | from starfleet.account_index.plugins.starfleet_default_index.ship import StarfleetDefaultAccountIndex 38 | 39 | ACCOUNT_INDEX_PLUGINS = [StarfleetDefaultAccountIndex] 40 | ``` 41 | 42 | ### Configuration 43 | For Starfleet to use your account index plugin, you need to have it configured to use it. At a minimum, you need to set the name of the plugin class in the `STARFLEET` configuration stanza's `AccountIndex` field. I.e. if your class is named `class FooAccountIndexPlugin`, then you will want to have `AccountIndex: FooAccountIndexPlugin` in your `STARFLEET` configuration stanza. 44 | 45 | As far as Marshmallow schemas are concerned, we do not require you to define one, however you are welcome to should you desire. If you do this, then add the logic in your plugin's `init()` function call. See the `StarfleetDefaultAccountIndex` plugin's code in `starfleet.account_index.plugins.starfleet_default_index` for details. 46 | 47 | ## Loader 48 | Account index plugins are loaded and exposed via the `ACCOUNT_INDEX` singleton. The account index plugins are registered and the _configured_ (see above) plugin is instantiated by the singleton by calling the `ACCOUNT_INDEX.index` property to get the loaded and ready-to-go account index object back out. 49 | 50 | The singleton has an pytest fixture named `test_index` in `tests.account_index.conftest` that overrides the path for the plugins to load plugins from `tests.account_index.testing_plugins`. 51 | 52 | This otherwise works exactly like the worker ship loader. 53 | 54 | !!! tip 55 | We recommend that if you make an account index plugin that it be as fast as possible. Reducing network IO would be desireable. But, also keep memory usage in mind since you don't want to run out of memory for your Lambda. 56 | -------------------------------------------------------------------------------- /mkdocs/userGuide/GitHubSync/Template.md: -------------------------------------------------------------------------------- 1 | # GitHub Repo Sync Worker: Payload Template Schema 2 | In this section, we discuss the GitHub Repo Sync worker templates. The GitHub Repo Sync worker is a `SINGLE_INVOCATION` worker, and does not require any AWS account on the account index. 3 | 4 | ## Template Schema 5 | Here is an example of the schema: 6 | ```yaml 7 | TemplateName: NameOfYourTemplate 8 | TemplateDescription: Syncs the REPO-NAME to the BUCKET-NAME 9 | Organization: YOUR-ORG-NAME-HERE 10 | Repository: THE-REPO-HERE 11 | BranchName: THE-BRANCH-NAME-HERE 12 | GitHubAppId: "The GitHub App Id -- need to wrap in quotes as it's technically a number but this needs to be a string" 13 | GitHubInstallationId: "The GitHub Installation ID -- ditto about wrapping in quotes" 14 | BucketName: YOUR-S3-BUCKET 15 | BucketRegion: S3 BUCKET REGION HERE 16 | ExtractZipContents: True # See below for details 17 | DeleteMissingFiles: True # See below for details 18 | 19 | # https://www.regexpal.com/ is your friend 20 | ExcludeRepoPaths: 21 | - "^testing\/.+$" # Excluding the `testing/` path 22 | - "^README.md$" 23 | - "^.github\/.+$" 24 | - "^.gitignore$" 25 | ``` 26 | 27 | Below are the **required** fields that this worker needs: 28 | 29 | 1. **`Organization`** - This is the name of the GitHub organization that the repository resides in. Note: This must be the same exact value (case sensitive) that is in the Secret to reference the GitHub App private key. 30 | 1. **`Repository`** - This is the name of the repository to sync. You need to make sure that the GitHub App has permissions to download this repo. 31 | 1. **`BranchName`** - This is the name of the branch that you want to sync. Typically, this would be `main` or `master`. 32 | 1. **`GitHubAppId`** - This is the *Application ID* (documented in the [GitHub App Configuration page](ConfigureGitHubApp.md)). Wrap it in quotes because it's a numerical string. 33 | 1. **`GitHubInstallationId`** - This is the *Installation ID* (documented in the [GitHub App Configuration page](ConfigureGitHubApp.md)). Wrap it in quotes because it's a numerical string. 34 | 1. **`BucketName`** - This is the name of the S3 bucket you want to sync with 35 | 1. **`BucketRegion`** - This is the region that the bucket resides in 36 | 1. **`ExtractZipContents`** - Boolean - This indicates whether or not the downloaded `.zip` file from GitHub should be extracted or not. That is to say, if this field is `False`, then the downloaded `.zip` file itself is what will be synced with S3. If this is set to `True`, then the `.zip` is extracted and the extracted files is what will be compared with S3. This being set to `False` is useful if you wanted to have the entire repository itself be the artifact that you want stored in S3 for downstream processing. For the use case of syncing payload templates to the templates S3 bucket, you want this to be set to `True` so that each template is individually synced with S3. 37 | 38 | Here are the **optional** fields: 39 | 40 | 1. **`IncludeRepoPaths`** - List of Regex Strings - This is a list of regex strings indicate the paths on the local disk that should be synced with S3. By default, this value is set to: `["^.+$"]`, which will match on everything. As mentioned on the previous page, if you wanted to sync the templates repository with S3, then for the testing deployment, you could have this set to `- "^testing\/.+$"`, which would only sync repository files that exist in the `testing\` path. 41 | 1. **`ExcludeRepoPaths`** - List of Regex Strings - This is a list of regex strings that indicate the paths on the local disk that should *not* be synced with S3. This takes precedence over the `IncludeRepoPaths`. In the included example above, we are ignoring specific files that we don't want synced with S3, like the `.gitignore` file. By default, this field is an empty list: `[]`. 42 | 1. **`KeyPrefix`** - This is the starting prefix in the S3 bucket that should be assessed. By default, the root of the S3 bucket is what the worker examines. If you wanted to only sync files that reside within the `foo/` prefix of the bucket, then you would want to set this value to `foo/`. All repo files would then be set to reside in `foo/filename...` on the bucket. 43 | 1. **`DeleteMissingFiles`** - Boolean - By default, this is set to `False`. If this is set to `True`, then the worker will *delete* any object in the S3 bucket that is *not* found in the GitHub repository. This flag being enabled keeps the S3 bucket contents 1:1 in sync with the repository. 44 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/aws_config/conftest.py: -------------------------------------------------------------------------------- 1 | """Pytest fixtures for the AWS Config Worker 2 | 3 | All the Pytest fixtures unique to the AWS Config worker 4 | 5 | :Module: starfleet.tests.starfleet_included_plugins.aws_config.conftest 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | # pylint: disable=unused-argument,unused-import,redefined-outer-name 12 | from typing import Any, Dict, Generator 13 | 14 | import boto3 15 | import pytest 16 | import yaml 17 | from botocore.client import BaseClient 18 | from moto import mock_aws 19 | 20 | from tests.account_index.conftest import test_index # noqa 21 | 22 | 23 | @pytest.fixture 24 | def aws_config(aws_sts: BaseClient) -> Generator[BaseClient, None, None]: 25 | """This is a fixture for a Moto wrapped AWS Config mock for the entire unit test. This also imports the STS mock.""" 26 | with mock_aws(): 27 | yield boto3.client("config", region_name="us-east-2") # Assuming that our deployment region for everything is us-east-2. 28 | 29 | 30 | @pytest.fixture() 31 | def template() -> Dict[str, Any]: 32 | """Template for describing what the state should be.""" 33 | return yaml.safe_load( 34 | """ 35 | TemplateName: AWSConfigEnablement 36 | TemplateDescription: Enabled AWS Config everywhere 37 | IncludeAccounts: 38 | AllAccounts: True 39 | OperateInOrgRoot: True 40 | IncludeRegions: 41 | - ALL 42 | AccountOverrideConfigurations: 43 | - 44 | IncludeAccounts: 45 | ByIds: 46 | - "000000000001" 47 | IncludeRegions: 48 | - us-west-1 49 | DeliveryChannelDetails: 50 | BucketName: bucket-000000000001 51 | BucketKeyPrefix: some/prefix/ 52 | S3DeliveryFrequency: TwentyFour_Hours 53 | S3KmsKeyArn: arn:aws:kms:us-west-1:000000000001:key/1234-1445-1919232 54 | SnsTopicArn: arn:aws:sns:us-west-1:000000000001:topic/sometopic 55 | PreferredName: us-west-1-000000000001 56 | RecorderConfiguration: 57 | PreferredName: us-west-1-000000000001 58 | ConfigRoleName: MyConfigRole 59 | RecordingEnabled: True 60 | RecordingGroup: 61 | RecordEverything: 62 | RecordGlobalsInTheseRegions: 63 | - us-west-1 64 | RetentionPeriodInDays: 30 65 | - 66 | IncludeAccounts: 67 | ByIds: 68 | - "000000000002" 69 | IncludeRegions: 70 | - us-west-1 71 | DeliveryChannelDetails: 72 | BucketName: bucket-000000000002 73 | S3DeliveryFrequency: TwentyFour_Hours 74 | RecorderConfiguration: 75 | ConfigRoleName: MyConfigRole 76 | RecordingEnabled: True 77 | RecordingGroup: 78 | RecordSpecificResources: 79 | - AWS::S3::Bucket 80 | - AWS::EC2::SecurityGroup 81 | RetentionPeriodInDays: 2557 82 | DefaultConfiguration: 83 | DeliveryChannelDetails: 84 | BucketName: all-bucket 85 | S3DeliveryFrequency: TwentyFour_Hours 86 | RecorderConfiguration: 87 | ConfigRoleName: MyConfigRole 88 | RecordingEnabled: True 89 | RecordingGroup: 90 | RecordEverything: 91 | RecordGlobalsInTheseRegions: 92 | - us-east-1 93 | RetentionPeriodInDays: 2557 94 | """ 95 | ) 96 | 97 | 98 | @pytest.fixture() 99 | def loaded_template(template: Dict[str, Any]) -> Dict[str, Any]: 100 | """This is the template loaded through the schema.""" 101 | from starfleet.worker_ships.plugins.aws_config.schemas import AwsConfigWorkerShipPayloadTemplate 102 | 103 | return AwsConfigWorkerShipPayloadTemplate().load(template) 104 | -------------------------------------------------------------------------------- /mkdocs/blog/posts/iam-role-worker.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2023-07-20 3 | authors: 4 | - mikegrima 5 | categories: 6 | - Workers 7 | - IAM 8 | - New Features 9 | --- 10 | 11 | # IAM Role Worker 12 | We have developed a Starfleet worker for syncing IAM roles. This worker wraps the excellent [IAMbic](https://iambic.org/) library to perform the heavy lifting of the actual IAM work. If you haven't checked out [IAMbic](https://iambic.org) yet, please take a look at it. It's a full-featured IaC that unifies IAM management across the AWS world (and other identity providers) with lots of features and capabilities. 13 | 14 | Most of the documentation for this worker resides in our [User Guide](../../userGuide/IAM/Roles/Overview.md). 15 | 16 | ## How does this work? 17 | The long and the short of it, we are simply embedding an IAMbic template within a Starfleet template. Starfleet performs the heavy lifting of tasking worker lambdas with the AWS account context to operate in. Starfleet then embeds the current account context within the IAMbic template that gets passed into the IAMbic library to sync the role. 18 | 19 | Here is a sample role template: 20 | 21 | ```yaml 22 | TemplateName: DevOpsAutomationRole 23 | TemplateDescription: This is a role for DevOps automation to do DevOps things 24 | IncludeAccounts: 25 | ByNames: 26 | - DevOpsTest 27 | - DevOpsProd 28 | IambicRoleTemplate: # <----- The IAMbic template gets embedded into here 29 | properties: 30 | role_name: DevOpsAutomationRole 31 | description: 'The DevOpsRole for DevOpsAutomation in {{ var.account_name }}' 32 | assume_role_policy_document: 33 | statement: 34 | - action: sts:AssumeRole 35 | effect: Allow 36 | principal: 37 | service: ec2.amazonaws.com 38 | version: '2012-10-17' 39 | managed_policies: 40 | - policy_arn: arn:aws:iam::aws:policy/ReadOnlyAccess 41 | inline_policies: 42 | - policy_name: DevOpsThings 43 | statement: 44 | - sid: DevOpsThings 45 | effect: Allow 46 | action: 47 | - ec2:* 48 | - elasticloadbalancing:* 49 | - iam:PassRole 50 | resource: '*' 51 | version: '2012-10-17' 52 | 53 | # Give the DevOpsTest account access to the DevOpsTest S3 Bucket: 54 | - StarfleetIncludeAccounts: 55 | ByNames: 56 | - DevOpsTest 57 | policy_name: ArtifactBucket 58 | statement: 59 | - effect: allow 60 | action: 61 | - s3:Get* 62 | - s3:List* 63 | - s3:PutObject 64 | resource: 65 | - aws:s3:::some-devops-bucket-devopstest 66 | - aws:s3:::some-devops-bucket-devopstest/* 67 | version: '2012-10-17' 68 | 69 | # Give the DevOpsProd account access to the DevOpsProd S3 Bucket: 70 | - StarfleetIncludeAccounts: 71 | ByNames: 72 | - DevOpsProd 73 | policy_name: ArtifactBucket 74 | statement: 75 | - effect: allow 76 | action: 77 | - s3:Get* 78 | - s3:List* 79 | - s3:PutObject 80 | resource: 81 | - aws:s3:::some-devops-bucket-devopsprod 82 | - aws:s3:::some-devops-bucket-devopsprod/* 83 | version: '2012-10-17' 84 | ``` 85 | 86 | ## What is different between this and vanilla IAMbic? 87 | Starfleet is simply wrapping the IAMbic library so that you use Starfleet instead of vanilla IAMbic. The primary benefit is that if you are already using Starfleet, then you can start rolling out IAM roles where you need it with drift prevention. You just need to familiarize yourself with the IAMbic template format to begin using it. 88 | 89 | There are however, some caveats. This is documented more in the [User Guide](../../userGuide/IAM/Roles/Overview.md). 90 | 91 | #### Other IAMbic features 92 | IAMbic has a lot of other standalone features, like support for other cloud provider identities, the ability to import existing IAM principals into IAMbic templates, and also the ability to implement time-limited policies that are not presently supported in Starfleet. The Starfleet worker is ideal if you don't have a need for those features and you just want to define an IAM role that must be consistently deployed in all the places you need it. 93 | 94 | ## Other IAM capabilities? 95 | At the time of writing, the IAM role worker is the main need we have, but we would love to add more IAM features in the future. IAMbic makes this really easy for us to do. Of course, if you would like to contribute, we would love to help! 96 | -------------------------------------------------------------------------------- /tests/starfleet_included_plugins/github_sync/test_auth.py: -------------------------------------------------------------------------------- 1 | """Tests for the GitHub Sync worker's GitHub authentication components 2 | 3 | Tests out the authentication code for GitHub. 4 | 5 | :Module: starfleet.tests.starfleet_included_plugins.github_sync.test_auth 6 | :Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | # pylint: disable=unused-argument,too-many-arguments,too-many-locals 12 | from datetime import datetime, timezone, UTC 13 | from typing import Any, Dict 14 | from unittest import mock 15 | from unittest.mock import MagicMock 16 | 17 | import jwt 18 | import pytest 19 | 20 | 21 | def test_make_app_token(unit_test_secrets: Dict[str, Any]) -> None: 22 | """This tests that we get a proper GitHub token back out""" 23 | from starfleet.worker_ships.plugins.github_sync.auth import GitHubAuthManager 24 | from tests.starfleet_included_plugins.github_sync.conftest import TEST_PUBLIC_KEY 25 | 26 | secret = unit_test_secrets["GitHubSyncWorker"]["fakeorg"] 27 | 28 | auth_manager = GitHubAuthManager() 29 | auth_manager._make_app_token("fakeorg", "1234567", secret) # noqa 30 | token = auth_manager._app_tokens["fakeorg"] 31 | 32 | payload = jwt.decode(token, TEST_PUBLIC_KEY, algorithms=["RS256"]) 33 | 34 | # Note breakpoints while debugging can cause issues with this due to timing. Make breakpoints for testing this after the "now =" 35 | now = int(datetime.now(tz=timezone.utc).timestamp()) 36 | assert payload["iss"] == "1234567" 37 | assert now - 65 <= payload["iat"] < now 38 | assert now + 301 >= payload["exp"] > now 39 | 40 | 41 | def test_make_installation_token(mock_installation_token: MagicMock) -> None: 42 | """This tests that we get a proper GitHub installation token out.""" 43 | from starfleet.worker_ships.plugins.github_sync.auth import GitHubAuthError, GitHubAuthManager 44 | 45 | auth_manager = GitHubAuthManager() 46 | auth_manager._app_tokens["fakeorg"] = "sometokenlol" 47 | auth_manager._make_installation_token("fakeorg", "987654") 48 | 49 | assert auth_manager._installation_tokens["fakeorg"]["token"] == "lolsometoken" 50 | assert auth_manager._installation_tokens["fakeorg"]["expiration"] > int(datetime.now(UTC).timestamp()) 51 | 52 | # Test with an invalid status code: 53 | mock_installation_token.post.return_value.status_code = 401 54 | with mock.patch("starfleet.worker_ships.plugins.github_sync.auth.LOGGER") as mocked_logger: 55 | with pytest.raises(GitHubAuthError): 56 | auth_manager._make_installation_token("fakeorg", "987654") 57 | 58 | assert "installation token: 401" in mocked_logger.error.call_args.args[0] 59 | 60 | 61 | def test_authenticate_github(mock_installation_token: MagicMock) -> None: 62 | """This tests the full authentication functionality and caching logic.""" 63 | from starfleet.worker_ships.plugins.github_sync.auth import GitHubAuthManager 64 | 65 | auth_manager = GitHubAuthManager() 66 | auth_token = auth_manager.authenticate("fakeorg", "1234567", "987654") 67 | assert auth_token == {"Authorization": "Bearer lolsometoken"} 68 | 69 | # Test the caching logic: 70 | with mock.patch("starfleet.worker_ships.plugins.github_sync.auth.LOGGER") as mocked_logger: 71 | assert auth_manager.authenticate("fakeorg", "1234567", "987654") == auth_token 72 | assert "Using cached credentials" in mocked_logger.debug.call_args.args[0] 73 | 74 | # Test the exceptions: 75 | mocked_logger.reset_mock() 76 | with pytest.raises(KeyError) as exc: 77 | auth_manager.authenticate("lolno", "1234567", "987654") 78 | assert exc.value.args[0] == "lolno" 79 | assert "organization: lolno" in mocked_logger.error.call_args.args[0] 80 | 81 | 82 | def test_github_auth_decorator(mock_installation_token: MagicMock) -> None: 83 | """This tests that the GitHub auth decorator works and properly injects credentials to functions wrapped with it.""" 84 | from starfleet.worker_ships.plugins.github_sync.auth import github_auth 85 | 86 | @github_auth 87 | def some_github_function(organization: str, some_other_arg: str, github_headers: Dict[str, str] = None, some_kwarg: str = None) -> None: 88 | assert organization == "fakeorg" 89 | assert github_headers == {"Authorization": "Bearer lolsometoken"} 90 | assert some_other_arg == "some other arg" 91 | assert some_kwarg == "some kwarg" 92 | 93 | some_github_function("fakeorg", "1234567", "987654", "some other arg", some_kwarg="some kwarg") # pylint: disable=redundant-keyword-arg 94 | -------------------------------------------------------------------------------- /src/starfleet/worker_ships/loader.py: -------------------------------------------------------------------------------- 1 | """Starfleet's worker "ship" loader. 2 | 3 | This does all the logic required to load Starfleet workers (which we call plugins 🚀for fun). 4 | 5 | :Module: starfleet.worker_ships.loader 6 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 7 | :License: See the LICENSE file for details 8 | :Author: Mike Grima 9 | """ 10 | 11 | from typing import Dict 12 | 13 | import starfleet.worker_ships.plugins 14 | from starfleet.utils.logging import LOGGER 15 | from starfleet.utils.configuration import BadConfigurationError, STARFLEET_CONFIGURATION 16 | from starfleet.utils.plugin_loader import find_plugins 17 | from starfleet.worker_ships.ship_schematics import StarfleetWorkerShip, StarfleetWorkerShipInstance 18 | 19 | 20 | class InvalidStarfleetWorkerException(Exception): 21 | """Exception raised if the Starfleet worker ship plugin is NOT a subclass of StarfleetWorkerShip""" 22 | 23 | 24 | class StarfleetWorkerShipLoader: 25 | """This will load all the Starfleet worker plugins.""" 26 | 27 | # These are defined here for easy testability: 28 | _worker_ship_path: str = starfleet.worker_ships.plugins.__path__ 29 | _worker_ship_prefix: str = starfleet.worker_ships.plugins.__name__ + "." 30 | 31 | def __init__(self): 32 | self._worker_ships: Dict[str, StarfleetWorkerShipInstance] = None # noqa 33 | 34 | def reset(self) -> None: 35 | """This resets the loader. This is only used as a convenience for unit testing.""" 36 | self._worker_ships = None 37 | 38 | def load_all_plugins(self): 39 | """ 40 | This will load all Starfleet worker ship plugins and verify that they are set up properly. This is code that will mostly be used by both the 41 | Starbase and the CLI. 42 | """ 43 | self._worker_ships = {} 44 | 45 | LOGGER.debug("[📦] Loading worker ship plugins...") 46 | try: 47 | for _, plugin_classes in find_plugins(self._worker_ship_path, self._worker_ship_prefix, "WORKER_SHIP_PLUGINS", StarfleetWorkerShip).items(): 48 | for plugin in plugin_classes: 49 | LOGGER.debug(f"[🔧] Configuring worker ship: {plugin.get_worker_ship_name()}") # noqa 50 | 51 | # Check if the worker has a configuration entry. If not then skip: 52 | worker_ship_config = STARFLEET_CONFIGURATION.config.get(plugin.get_worker_ship_name()) # noqa 53 | if worker_ship_config: 54 | # If there is a configuration entry, then we need to validate the correct configuration: 55 | errors = plugin.configuration_template_class().validate(worker_ship_config) # noqa 56 | if errors: 57 | raise BadConfigurationError( 58 | f"[💥] Worker ship: {plugin.get_worker_ship_name()} has an invalid configuration. {str(errors)}" 59 | ) # noqa 60 | 61 | # Check that the worker ship is enabled: 62 | if not worker_ship_config["Enabled"]: 63 | LOGGER.debug(f"[⏭️] Worker ship: {plugin.get_worker_ship_name()} is DISABLED in it's configuration. Skipping...") # noqa 64 | continue 65 | 66 | # Instantiate the worker class: 67 | self._worker_ships[plugin.get_worker_ship_name()] = plugin() # noqa 68 | LOGGER.debug(f"[👍] Worker ship: {plugin.get_worker_ship_name()} is properly configured and ENABLED.") # noqa 69 | 70 | else: 71 | LOGGER.debug(f"[⏭️] Worker ship: {plugin.get_worker_ship_name()} has no discovered configuration. Skipping... ") # noqa 72 | continue 73 | 74 | except Exception as exc: 75 | LOGGER.error("[💥] Major exception encountered configuring all the Starfleet worker ship plugins. See the stacktrace for details.") 76 | LOGGER.exception(exc) 77 | raise 78 | 79 | if not self._worker_ships: 80 | LOGGER.debug("[🤷] There were no properly enabled worker ships to load") 81 | else: 82 | LOGGER.debug(f"[🚀] Completed loading {len(self._worker_ships)} worker ships") 83 | 84 | def get_worker_ships(self) -> Dict[str, StarfleetWorkerShipInstance]: 85 | """This will return all the worker ship instances. This is used by the Starbase and by the CLI.""" 86 | if self._worker_ships is None: 87 | self.load_all_plugins() 88 | 89 | return self._worker_ships 90 | 91 | 92 | STARFLEET_WORKER_SHIPS = StarfleetWorkerShipLoader() 93 | -------------------------------------------------------------------------------- /src/starfleet/utils/config_schema.py: -------------------------------------------------------------------------------- 1 | """Starfleet's configuration schema 2 | 3 | This defines a basic Marshmallow schema for the Starfleet configuration. This will ensure that the base configuration file 4 | has the correct components on it. 5 | 6 | :Module: starfleet.utils.config_schema 7 | :Copyright: (c) 2022 by Gemini Trust Company, LLC., see AUTHORS for more info 8 | :License: See the LICENSE file for details 9 | :Author: Mike Grima 10 | """ 11 | 12 | from typing import Any, Dict 13 | 14 | from marshmallow import Schema, fields, INCLUDE, validate, validates_schema, ValidationError 15 | 16 | from starfleet.utils.niceties import get_all_regions 17 | 18 | aws_regions = get_all_regions() 19 | 20 | 21 | class SecretsManager(Schema): 22 | """This is a nested schema for AWS Secrets manager which is a pair of the Secrets ID and the region it resides in.""" 23 | 24 | secret_id = fields.String(required=True, data_key="SecretId") 25 | 26 | # We are assuming that the Secrets Manager regions are the same as EC2 27 | secret_region = fields.String(required=True, validate=validate.OneOf(aws_regions), data_key="SecretRegion") 28 | 29 | 30 | class StarfleetSchema(Schema): 31 | """This is the main schema for Starfleet itself.""" 32 | 33 | # Required Fields: 34 | # This is where all Starfleet resources (SQS, S3, etc.) reside. 35 | deployment_region = fields.String(required=True, data_key="DeploymentRegion", validate=validate.OneOf(aws_regions)) 36 | template_bucket = fields.String(required=True, data_key="TemplateBucket") # This is the name of the S3 bucket that all the templates will reside. 37 | # This is the SQS queue URL that the Starbase will use for getting the worker/template details so that the worker ship can be tasked properly: 38 | fanout_queue_url = fields.Url(required=True, schemes={"https"}, data_key="FanOutQueueUrl") 39 | 40 | # This is the name for the account index ship plugin. The default should be used if you are using the AccountIndexGeneratorShip worker ship: 41 | account_index = fields.String(required=False, data_key="AccountIndex", load_default="StarfleetDefaultAccountIndex") 42 | 43 | # Optional fields: 44 | # This is a field that limits the ACCOUNT_REGION workers such that there are specific regions that can be operated on. 45 | # If this is set, then you can only run in the regions defined here despite what regions an account has enabled: 46 | scope_to_regions = fields.List(fields.String(validate=validate.OneOf(aws_regions)), required=False, data_key="ScopeToRegions", load_default=[]) 47 | # ^^ This is useful if you have an SCP that disables regions; this prevents Starfleet to run in regions that are disabled by SCP. 48 | 49 | # Secrets Manager ARN for Starfleet's secrets if required 50 | secrets_manager = fields.Nested(SecretsManager(), required=False, data_key="SecretsManager") 51 | 52 | # If we want Slack alerts, then you need to set this to True (which channel ID to use is set for each worker configuration) 53 | # and the API token is stored in the SecretsManager 54 | slack_enabled = fields.Boolean(required=False, load_default=False, data_key="SlackEnabled") 55 | 56 | # Log Level: 57 | log_level = fields.String( 58 | required=False, load_default="INFO", validate=validate.OneOf({"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"}), data_key="LogLevel" 59 | ) 60 | # Dictionary to override log levels for 3rd party loggers. This is the name of the log and the level. 61 | third_party_logger_levels = fields.Dict(required=False, data_key="ThirdPartyLoggerLevels") 62 | 63 | @validates_schema(pass_original=True) 64 | def verify_schema(self, data: Dict[str, Any], original_data: Dict[str, Any], **kwargs) -> None: # pylint: disable=unused-argument # noqa 65 | """ 66 | This validates that the schema is correct. At present, this is going to validate: 67 | 1. That if SlackEnabled is True, that we also have SecretsManager configured. 68 | """ 69 | errors = {} 70 | if data.get("slack_enabled"): # Check the properly parsed. 71 | # Verify that we also have Secrets Manager: 72 | if not data.get("secrets_manager"): 73 | errors["SlackEnabled"] = ["Slack can only be enabled if you are using Secrets Manager -- you need to configure the `SecretsManager` field."] 74 | 75 | if errors: 76 | raise ValidationError(errors) 77 | 78 | 79 | class BaseConfigurationSchema(Schema): 80 | """The base configuration Schema for Starfleet""" 81 | 82 | # Required fields: 83 | starfleet = fields.Nested(StarfleetSchema, required=True, data_key="STARFLEET") 84 | 85 | class Meta: 86 | """Meta properties on the Schema used by Marshmallow""" 87 | 88 | unknown = INCLUDE # It's totally OK and normal if we get values that are not in this schema -- we only care that we got the required values 89 | -------------------------------------------------------------------------------- /mkdocs/userGuide/GitHubSync/CLI.md: -------------------------------------------------------------------------------- 1 | # GitHub Repo Sync Worker CLI 2 | 3 | The GitHub Repo Sync worker has 3 commands. All commands require AWS credentials used by Starfleet to be exported into the environment. All commands are under the `sync-github` sub-command. 4 | 5 | All commands require a `--payload` parameter to point to the path of the YAML payload. 6 | 7 | ```bash 8 | starfleet sync-github 9 | Usage: starfleet sync-github [OPTIONS] COMMAND [ARGS]... 10 | 11 | This is the worker ship for syncing a GitHub repo with S3. 12 | 13 | Options: 14 | --help Show this message and exit. 15 | 16 | Commands: 17 | download This is a helpful debugging command to download... 18 | get-installation-token This returns the installation token for the... 19 | run This will run the syncing of the repository... 20 | ``` 21 | 22 | ## The `get-installation-token` command 23 | The `get-installation-token` command is used for debugging purposes. It outputs an ephemeral GitHub authorization bearer token for the GitHub App specified in the payload template. This token can be used to make authenticated API calls to GitHub. You can use this with curl or Postman to debug the GitHub App's ability to make authenticated calls to GitHub. 24 | 25 | Help text: 26 | ```bash 27 | starfleet sync-github get-installation-token --help 28 | Usage: starfleet sync-github get-installation-token [OPTIONS] 29 | 30 | This returns the installation token for the given organization, application 31 | id, and installation id provided in the payload template. This is mostly 32 | used for local testing and debugging. 33 | 34 | Options: 35 | --payload FILENAME This is the worker payload YAML [required] 36 | --commit Must be supplied for changes to be made [does not do anything] 37 | --help Show this message and exit. 38 | ``` 39 | 40 | The `--commit` flag does not have any purpose for this command. 41 | 42 | Here is a sample command that would generate the token: 43 | ```bash 44 | starfleet sync-github get-installation-token --payload /path/to/payload.yaml 45 | ``` 46 | 47 | ## The `download` command 48 | The `download` command simply downloads the GitHub repository to the local disk. It will extract the `.zip` file downloaded from GitHub if the payload specifies the `ExtractZipContents` flag set to `True`. This is mostly used for debugging purposes. 49 | 50 | Help text: 51 | ```bash 52 | starfleet sync-github download --help 53 | Usage: starfleet sync-github download [OPTIONS] 54 | 55 | This is a helpful debugging command to download the repository. This will 56 | extract the contents if the payload specifies the `ExtractZipContents` flag 57 | set to `True`. You simply provide the payload template and the location for 58 | where you want the repo to be downloaded (and optionally extracted), and it 59 | will be saved to that path as `REPO_NAME.zip` (and extracted as `REPO_NAME- 60 | COMMIT-HASH/`). 61 | 62 | The commit flag doesn't do anything for this command. 63 | 64 | Options: 65 | --save-dir PATH A local directory to save the zip in. [required] 66 | --payload FILENAME This is the worker payload YAML [required] 67 | --commit Must be supplied for changes to be made [does not do anything] 68 | --help Show this message and exit. 69 | ``` 70 | 71 | The `--commit` flag does not have any purpose for this command. 72 | 73 | Here is a sample command that would save the repo to the Desktop: 74 | ```bash 75 | starfleet sync-github get-installation-token --payload /path/to/payload.yaml --save-dir ~/Desktop/ 76 | ``` 77 | 78 | ## The `run` command 79 | The `run` command performs the full function of the workload where it will download the repo, optionally extract the contents, and attempt to sync with S3. This works the same way the Lambda function in the cloud would. This optionally allows you to specify a download directory to persist the downloaded components. 80 | 81 | Help text: 82 | ```bash 83 | starfleet sync-github run --help 84 | Usage: starfleet sync-github run [OPTIONS] 85 | 86 | This will run the syncing of the repository against the payload's specified 87 | S3 bucket. 88 | 89 | Options: 90 | --save-dir PATH An optional local directory to save and retain the 91 | contents within. If not supplied, then this will create 92 | a temporary directory and delete it. 93 | --payload FILENAME This is the worker payload YAML [required] 94 | --commit Must be supplied for changes to be made 95 | --help Show this message and exit. 96 | ``` 97 | 98 | The `--commit` flag will attempt to make changes to the S3 bucket's files if there is a change that needs to run. 99 | 100 | Here is a sample command that would save the repo to the Desktop, and also attempt to sync the changes to S3. Commit enabled: 101 | ```bash 102 | starfleet sync-github run --payload /path/to/payload.yaml --save-dir ~/Desktop/ --commit 103 | ``` 104 | --------------------------------------------------------------------------------