├── core
├── __init__.py
├── abstract_service_pipeline.py
├── pipelines_stack.py
├── monorepo_stack.py
└── lambda
│ └── handler.py
├── .pep8
├── monorepo-sample
├── README.md
├── demo
│ ├── error.html
│ └── index.html
├── hotsite
│ ├── error.html
│ └── index.html
└── monorepo-main.json
├── cdk.json
├── create-requirement.sh
├── list-outdated-cdk-libs.sh
├── docs
├── architecture.jpg
└── monorepo-stacks.jpg
├── upgrade-cdk-libs.sh
├── .gitignore
├── requirements.txt
├── app.py
├── CODE_OF_CONDUCT.md
├── monorepo_config.py
├── LICENSE
├── setup.py
├── Makefile
├── CONTRIBUTING.md
├── pipelines
├── pipeline_demo.py
└── pipeline_hotsite.py
└── README.md
/core/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/.pep8:
--------------------------------------------------------------------------------
1 | [pycodestyle]
2 | max_line_length = 120
--------------------------------------------------------------------------------
/monorepo-sample/README.md:
--------------------------------------------------------------------------------
1 | This is a mono-repository for testing purposes!
--------------------------------------------------------------------------------
/cdk.json:
--------------------------------------------------------------------------------
1 | {
2 | "app": "python3 app.py",
3 | "context": {
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/monorepo-sample/demo/error.html:
--------------------------------------------------------------------------------
1 |
2 | Error!
3 |
--------------------------------------------------------------------------------
/create-requirement.sh:
--------------------------------------------------------------------------------
1 | pip list --format=freeze --exclude-editable > requirement.txt
2 |
--------------------------------------------------------------------------------
/monorepo-sample/hotsite/error.html:
--------------------------------------------------------------------------------
1 |
2 | Error!
3 |
--------------------------------------------------------------------------------
/list-outdated-cdk-libs.sh:
--------------------------------------------------------------------------------
1 | pip list --exclude-editable --outdated | grep cdk | awk '{print $1}'
2 |
--------------------------------------------------------------------------------
/docs/architecture.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aws-samples/monorepo-multi-pipeline-trigger/HEAD/docs/architecture.jpg
--------------------------------------------------------------------------------
/docs/monorepo-stacks.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aws-samples/monorepo-multi-pipeline-trigger/HEAD/docs/monorepo-stacks.jpg
--------------------------------------------------------------------------------
/monorepo-sample/monorepo-main.json:
--------------------------------------------------------------------------------
1 | {
2 | "demo": "codepipeline-demo-main",
3 | "hotsite": "codepipeline-hotsite-main"
4 | }
--------------------------------------------------------------------------------
/upgrade-cdk-libs.sh:
--------------------------------------------------------------------------------
1 | pip list --outdated --format=freeze --exclude-editable | grep cdk | awk '{print $1}' | xargs -n1 pip install -U
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.swp
2 | package-lock.json
3 | __pycache__
4 | .pytest_cache
5 | .env
6 | .venv
7 | *.egg-info
8 |
9 | # CDK asset staging directory
10 | .cdk.staging
11 | cdk.out
12 |
--------------------------------------------------------------------------------
/monorepo-sample/hotsite/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | A simple hotsite!
6 |
7 |
8 |
9 | A simple hotsite!
10 |
11 |
12 |
--------------------------------------------------------------------------------
/monorepo-sample/demo/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | A simple demo website!
6 |
7 |
8 |
9 | A simple demo website!
10 |
11 |
12 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | attrs==21.4.0
2 | aws-cdk-lib==2.104.0
3 | cattrs==22.1.0
4 | constructs==10.3.0
5 | exceptiongroup==1.1.3
6 | jsii==1.91.0
7 | pip==23.3.1
8 | publication==0.0.3
9 | python-dateutil==2.8.2
10 | setuptools==70.0.0
11 | six==1.16.0
12 | typing_extensions==4.8.0
13 |
--------------------------------------------------------------------------------
/app.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | from aws_cdk import App
4 | from core.monorepo_stack import MonorepoStack
5 | from core.pipelines_stack import PipelineStack
6 |
7 | app = App()
8 | core = MonorepoStack(app, "MonoRepoStack")
9 | PipelineStack(app, "PipelinesStack", core.exported_monorepo)
10 |
11 | app.synth()
12 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | ## Code of Conduct
2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
4 | opensource-codeofconduct@amazon.com with any additional questions or comments.
5 |
--------------------------------------------------------------------------------
/core/abstract_service_pipeline.py:
--------------------------------------------------------------------------------
1 | from abc import ABC, abstractmethod
2 | from aws_cdk import (aws_codecommit as codecommit)
3 | from constructs import Construct
4 |
5 |
6 | class ServicePipeline(ABC):
7 |
8 | @abstractmethod
9 | def pipeline_name(self) -> str:
10 | pass
11 |
12 | @abstractmethod
13 | def build_pipeline(self, scope: Construct, code_commit: codecommit.Repository, pipeline_name: str, service_name: str):
14 | pass
--------------------------------------------------------------------------------
/core/pipelines_stack.py:
--------------------------------------------------------------------------------
1 | from aws_cdk import (Stack,
2 | aws_codecommit as codecommit)
3 | from constructs import Construct
4 | import monorepo_config
5 |
6 | class PipelineStack(Stack):
7 | def __init__(self, scope: Construct, construct_id: str, codecommit: codecommit, **kwargs) -> None:
8 | super().__init__(scope, construct_id, **kwargs)
9 | for dir_name, service_pipeline in monorepo_config.service_map.items():
10 | service_pipeline.build_pipeline(self, codecommit, service_pipeline.pipeline_name(), dir_name)
11 |
--------------------------------------------------------------------------------
/monorepo_config.py:
--------------------------------------------------------------------------------
1 | # This is a configuration file is used by PipelineStack to determine which pipelines should be constructed
2 |
3 | from core.abstract_service_pipeline import ServicePipeline
4 | from typing import Dict
5 |
6 |
7 | # Pipeline definition imports
8 | from pipelines.pipeline_demo import DemoPipeline
9 | from pipelines.pipeline_hotsite import HotsitePipeline
10 |
11 | ### Add your pipeline configuration here
12 | service_map: Dict[str, ServicePipeline] = {
13 | # folder-name -> pipeline-class
14 | 'demo': DemoPipeline(),
15 | 'hotsite': HotsitePipeline()
16 | }
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 |
3 | Permission is hereby granted, free of charge, to any person obtaining a copy of
4 | this software and associated documentation files (the "Software"), to deal in
5 | the Software without restriction, including without limitation the rights to
6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
7 | the Software, and to permit persons to whom the Software is furnished to do so.
8 |
9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
15 |
16 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import setuptools
2 |
3 |
4 | with open("README.md") as fp:
5 | long_description = fp.read()
6 |
7 |
8 | setuptools.setup(
9 | name="monorepo_codepipeline_trigger",
10 | version="0.0.1",
11 |
12 | description="CodeCommit monorepo multi pipeline triggers",
13 | long_description=long_description,
14 | long_description_content_type="text/markdown",
15 |
16 | author="author",
17 |
18 | package_dir={"": "."},
19 | packages=setuptools.find_packages(where="monorepo_codepipeline_trigger"),
20 |
21 | install_requires=[
22 | "aws-cdk-lib>=2.0.0",
23 | "constructs>=10.0.0"
24 | ],
25 |
26 | python_requires=">=3.6",
27 |
28 | classifiers=[
29 | "Development Status :: 4 - Beta",
30 |
31 | "Intended Audience :: Developers",
32 |
33 | "Programming Language :: JavaScript",
34 | "Programming Language :: Python :: 3 :: Only",
35 | "Programming Language :: Python :: 3.6",
36 | "Programming Language :: Python :: 3.7",
37 | "Programming Language :: Python :: 3.8",
38 |
39 | "Topic :: Software Development :: Code Generators",
40 | "Topic :: Utilities",
41 |
42 | "Typing :: Typed",
43 | ],
44 | )
45 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY: build package changes deploy-core deploy-pipelines deploy destroy-core destroy-pipelines destroy
2 |
3 | # define the name of the virtual environment directory
4 | VENV := .venv
5 | VENV_ACTIVATE := .venv/bin/activate
6 |
7 | # default target, when make is executed without arguments: install virtualenv and project dependencies
8 | all: install
9 |
10 | # Create virtualenv and install dependencies
11 | install:
12 | @( \
13 | python3 -m venv $(VENV); \
14 | source $(VENV_ACTIVATE); \
15 | pip install -r requirements.txt; \
16 | \
17 | )
18 |
19 | # Bootstraping the cdk
20 | bootstrap:
21 | ifeq ("$(account-id)","")
22 | @echo "Error: account-id parameter is mandatory\n"
23 | @exit 1
24 | endif
25 | ifeq ("$(region)","")
26 | @echo "Error: region parameter is mandatory\n"
27 | @exit 1
28 | endif
29 | @( \
30 | source $(VENV_ACTIVATE); \
31 | cdk bootstrap aws://${account-id}/${region}; \
32 | \
33 | )
34 |
35 | # Deploy monorepo core stack
36 | deploy-core :
37 | ifneq ("$(monorepo-name)","")
38 | $(eval params_monorepo := --parameters MonorepoName=$(monorepo-name))
39 | endif
40 | @( \
41 | source $(VENV_ACTIVATE); \
42 | echo cdk deploy MonoRepoStack ${params_monorepo}; \
43 | cdk deploy MonoRepoStack ${params_monorepo}; \
44 | \
45 | )
46 |
47 | # Deploy pipelines stack
48 | deploy-pipelines:
49 | @( \
50 | source $(VENV_ACTIVATE); \
51 | cdk deploy PipelinesStack; \
52 | \
53 | )
54 |
55 | # Deploy both stacks
56 | deploy: deploy-core deploy-pipelines
57 |
58 | # Destroy MonoRepo core stack
59 | destroy-core:
60 | @( \
61 | source $(VENV_ACTIVATE); \
62 | cdk destroy MonoRepoStack; \
63 | \
64 | )
65 |
66 | # Destroy Pipelines stack
67 | destroy-pipelines:
68 | @( \
69 | source $(VENV_ACTIVATE); \
70 | cdk destroy PipelinesStack; \
71 | \
72 | )
73 |
74 | # Remove virtual env files
75 | clean-files:
76 | rm -rf $(VENV)
77 | find . -type f -name '*.pyc' -delete
78 |
79 | # Destroy all
80 | destroy: destroy-pipelines destroy-core clean-files
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing Guidelines
2 |
3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional
4 | documentation, we greatly value feedback and contributions from our community.
5 |
6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary
7 | information to effectively respond to your bug report or contribution.
8 |
9 |
10 | ## Reporting Bugs/Feature Requests
11 |
12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features.
13 |
14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already
15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful:
16 |
17 | * A reproducible test case or series of steps
18 | * The version of our code being used
19 | * Any modifications you've made relevant to the bug
20 | * Anything unusual about your environment or deployment
21 |
22 |
23 | ## Contributing via Pull Requests
24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:
25 |
26 | 1. You are working against the latest source on the *main* branch.
27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.
28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted.
29 |
30 | To send us a pull request, please:
31 |
32 | 1. Fork the repository.
33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change.
34 | 3. Ensure local tests pass.
35 | 4. Commit to your fork using clear commit messages.
36 | 5. Send us a pull request, answering any default questions in the pull request interface.
37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.
38 |
39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and
40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/).
41 |
42 |
43 | ## Finding contributions to work on
44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start.
45 |
46 |
47 | ## Code of Conduct
48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
50 | opensource-codeofconduct@amazon.com with any additional questions or comments.
51 |
52 |
53 | ## Security issue notifications
54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue.
55 |
56 |
57 | ## Licensing
58 |
59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
60 |
--------------------------------------------------------------------------------
/core/monorepo_stack.py:
--------------------------------------------------------------------------------
1 | from aws_cdk import (Stack, RemovalPolicy, Duration, CfnParameter,
2 | aws_lambda as lambda_,
3 | aws_codecommit as codecommit,
4 | aws_iam as iam,
5 | aws_s3 as s3,
6 | aws_s3_deployment as s3_deployment)
7 | from constructs import Construct
8 | import os
9 | import zipfile
10 | import tempfile
11 | import json
12 |
13 |
14 | class MonorepoStack(Stack):
15 | exported_monorepo: codecommit.Repository
16 |
17 | def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
18 | super().__init__(scope, construct_id, **kwargs)
19 |
20 | ### Stack Parameters ###
21 | monorepo_name = CfnParameter(self, 'MonorepoName',
22 | type='String',
23 | description='CodeCommit Monorepo name',
24 | default='monorepo-sample')
25 |
26 | branch_for_trigger = 'main'
27 |
28 | function_name = f'{monorepo_name.value_as_string}-codecommit-handler'
29 | repository_name = monorepo_name.value_as_string
30 | region = Stack.of(self).region
31 | account = Stack.of(self).account
32 |
33 |
34 | monorepo = self.create_codecommit_repo(repository_name, branch_for_trigger)
35 |
36 | monorepo_lambda = self.create_lambda(region, account, repository_name, function_name)
37 |
38 | monorepo.grant_read(monorepo_lambda)
39 | monorepo.notify(f"arn:aws:lambda:{region}:{account}:function:{function_name}",
40 | name="lambda-codecommit-event", branches=[branch_for_trigger])
41 | self.exported_monorepo = monorepo
42 |
43 |
44 | def create_lambda(self, region, account, repository_name, function_name):
45 | # Lambda function which triggers code pipeline according
46 | # Function must run with concurrency = 1 -- to avoid race condition
47 | monorepo_lambda = lambda_.Function(self, "CodeCommitEventHandler",
48 | function_name=function_name,
49 | runtime=lambda_.Runtime.PYTHON_3_11,
50 | code=lambda_.Code.from_asset("core/lambda/"),
51 | handler="handler.main",
52 | timeout=Duration.seconds(60),
53 | dead_letter_queue_enabled=True,
54 | reserved_concurrent_executions=1)
55 | monorepo_lambda.add_permission("codecommit-permission",
56 | principal=iam.ServicePrincipal("codecommit.amazonaws.com"),
57 | action="lambda:InvokeFunction",
58 | source_arn=f"arn:aws:codecommit:{region}:{account}:{repository_name}")
59 | monorepo_lambda.add_to_role_policy(
60 | iam.PolicyStatement(resources=[f'arn:aws:ssm:{region}:{account}:parameter/MonoRepoTrigger/*'],
61 | actions=['ssm:GetParameter', 'ssm:GetParameters', 'ssm:PutParameter']))
62 | monorepo_lambda.add_to_role_policy(
63 | iam.PolicyStatement(resources=[f'arn:aws:codepipeline:{region}:{account}:*'],
64 | actions=['codepipeline:GetPipeline', 'codepipeline:ListPipelines',
65 | 'codepipeline:StartPipelineExecution', 'codepipeline:StopPipelineExecution']))
66 | return monorepo_lambda
67 |
68 |
69 | def create_codecommit_repo(self, repository_name, branch_for_trigger):
70 | tmp_dir = zip_sample()
71 | sample_bucket = s3.Bucket(self, 'MonoRepoSample',
72 | removal_policy=RemovalPolicy.DESTROY,
73 | auto_delete_objects=True)
74 | sample_deployment = s3_deployment.BucketDeployment(self, 'DeployMonoRepoSample',
75 | sources=[s3_deployment.Source.asset(tmp_dir)],
76 | destination_bucket=sample_bucket)
77 | monorepo = codecommit.Repository(self, "monorepo", repository_name=repository_name)
78 | cfn_repo = monorepo.node.find_child('Resource')
79 | cfn_repo.code = codecommit.CfnRepository.CodeProperty(s3={'bucket': sample_bucket.bucket_name, 'key': 'sample.zip'},
80 | branch_name=branch_for_trigger)
81 | monorepo.node.add_dependency(sample_deployment)
82 | return monorepo
83 |
84 |
85 | def zip_sample():
86 | # codepipeline_map = {}
87 | # for dir_name, service_pipeline in monorepo_config.service_map.items():
88 | # codepipeline_map[dir_name] = service_pipeline.pipeline_name()
89 | tempdir = tempfile.mkdtemp('bucket-sample')
90 | with zipfile.ZipFile(os.path.join(tempdir, 'sample.zip'), 'w') as zf:
91 | # zf.writestr(f'monorepo-{branch_name}.json', json.dumps(codepipeline_map))
92 | for dirname, subdirs, files in os.walk("./monorepo-sample/"):
93 | for filename in files:
94 | relativepath = os.path.join(dirname.replace("./monorepo-sample/", ""), filename)
95 | zf.write(os.path.join(dirname, filename), arcname=relativepath)
96 | return tempdir
97 |
--------------------------------------------------------------------------------
/pipelines/pipeline_demo.py:
--------------------------------------------------------------------------------
1 | from core.abstract_service_pipeline import ServicePipeline
2 | from aws_cdk import (RemovalPolicy, CfnOutput,
3 | aws_iam as iam,
4 | aws_codebuild as codebuild,
5 | aws_codepipeline as codepipeline,
6 | aws_codepipeline_actions as codepipeline_actions,
7 | aws_codecommit as codecommit,
8 | aws_s3 as s3,
9 | aws_cloudfront as cloudfront)
10 | from constructs import Construct
11 |
12 |
13 | class DemoPipeline(ServicePipeline):
14 |
15 | def pipeline_name(self) -> str:
16 | return 'codepipeline-demo-main'
17 |
18 | def build_pipeline(self, scope: Construct, code_commit: codecommit.Repository, pipeline_name: str, service_name: str):
19 | select_artifact_build = codebuild.PipelineProject(scope, f'SelectArtifactBuild-{pipeline_name}',
20 | build_spec=codebuild.BuildSpec.from_object(dict(
21 | version='0.2',
22 | phases=dict(
23 | build=dict(
24 | commands=[
25 | f'echo selecting directory for {service_name}...'])),
26 | artifacts={
27 | 'base-directory': service_name,
28 | 'files': ['**/*']})),
29 | environment=dict(build_image=codebuild.LinuxBuildImage.STANDARD_5_0))
30 | source_output = codepipeline.Artifact()
31 | service_artifact = codepipeline.Artifact()
32 |
33 | bucket = s3.Bucket(
34 | scope,
35 | f'Bucket-{pipeline_name}-{service_name}',
36 | block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
37 | website_index_document='index.html',
38 | website_error_document='error.html',
39 | auto_delete_objects=True,
40 | removal_policy=RemovalPolicy.DESTROY)
41 |
42 | cloudfront_oai = cloudfront.OriginAccessIdentity(scope, f"Cloudfront OAI for {service_name}")
43 |
44 | bucket.add_to_resource_policy(iam.PolicyStatement(
45 | effect=iam.Effect.ALLOW,
46 | actions=["s3:GetObject"],
47 | principals=[iam.CanonicalUserPrincipal(
48 | cloudfront_oai.cloud_front_origin_access_identity_s3_canonical_user_id)],
49 | resources=[bucket.arn_for_objects("*")]
50 | ))
51 |
52 | s3_origin_source = cloudfront.S3OriginConfig(s3_bucket_source=bucket, origin_access_identity=cloudfront_oai)
53 | source_config = cloudfront.SourceConfiguration(s3_origin_source=s3_origin_source, behaviors=[
54 | cloudfront.Behavior(is_default_behavior=True)])
55 |
56 | dist = cloudfront.CloudFrontWebDistribution(scope,
57 | service_name,
58 | origin_configs=[source_config],
59 | comment='CDK created',
60 | default_root_object="index.html")
61 |
62 | CfnOutput(scope, f'{service_name}_url', value=dist.distribution_domain_name,
63 | export_name=f'{service_name}-url')
64 |
65 | return codepipeline.Pipeline(scope, pipeline_name,
66 | pipeline_name=pipeline_name,
67 | stages=[
68 | codepipeline.StageProps(stage_name="Source",
69 | actions=[
70 | codepipeline_actions.CodeCommitSourceAction(
71 | action_name="CodeCommit_Source",
72 | branch="main",
73 | repository=code_commit,
74 | output=source_output,
75 | trigger=codepipeline_actions.CodeCommitTrigger.NONE)]),
76 | codepipeline.StageProps(stage_name="Build",
77 | actions=[
78 | codepipeline_actions.CodeBuildAction(
79 | action_name="SelectServiceArtifact",
80 | project=select_artifact_build,
81 | input=source_output,
82 | outputs=[service_artifact])]),
83 | codepipeline.StageProps(stage_name="Deploy",
84 | actions=[codepipeline_actions.S3DeployAction(action_name="DeployS3", bucket=bucket, input=service_artifact)]), ])
85 |
--------------------------------------------------------------------------------
/pipelines/pipeline_hotsite.py:
--------------------------------------------------------------------------------
1 | from core.abstract_service_pipeline import ServicePipeline
2 | from aws_cdk import (RemovalPolicy, CfnOutput,
3 | aws_iam as iam,
4 | aws_codebuild as codebuild,
5 | aws_codepipeline as codepipeline,
6 | aws_codepipeline_actions as codepipeline_actions,
7 | aws_codecommit as codecommit,
8 | aws_s3 as s3,
9 | aws_cloudfront as cloudfront)
10 | from constructs import Construct
11 |
12 |
13 | class HotsitePipeline(ServicePipeline):
14 |
15 | def pipeline_name(self) -> str:
16 | return 'codepipeline-hotsite-main'
17 |
18 | def build_pipeline(self, scope: Construct, code_commit: codecommit.Repository, pipeline_name: str, service_name: str):
19 | select_artifact_build = codebuild.PipelineProject(scope, f'SelectArtifactBuild-{pipeline_name}',
20 | build_spec=codebuild.BuildSpec.from_object(dict(
21 | version='0.2',
22 | phases=dict(
23 | build=dict(
24 | commands=[
25 | f'echo selecting directory for {service_name}...'])),
26 | artifacts={
27 | 'base-directory': service_name,
28 | 'files': ['**/*']})),
29 | environment=dict(build_image=codebuild.LinuxBuildImage.STANDARD_5_0))
30 | source_output = codepipeline.Artifact()
31 | service_artifact = codepipeline.Artifact()
32 |
33 | bucket = s3.Bucket(
34 | scope,
35 | f'Bucket-{pipeline_name}-{service_name}',
36 | block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
37 | website_index_document='index.html',
38 | website_error_document='error.html',
39 | auto_delete_objects=True,
40 | removal_policy=RemovalPolicy.DESTROY)
41 |
42 | cloudfront_oai = cloudfront.OriginAccessIdentity(scope, f"Cloudfront OAI for {service_name}")
43 |
44 | bucket.add_to_resource_policy(iam.PolicyStatement(
45 | effect=iam.Effect.ALLOW,
46 | actions=["s3:GetObject"],
47 | principals=[iam.CanonicalUserPrincipal(
48 | cloudfront_oai.cloud_front_origin_access_identity_s3_canonical_user_id)],
49 | resources=[bucket.arn_for_objects("*")]
50 | ))
51 |
52 | s3_origin_source = cloudfront.S3OriginConfig(s3_bucket_source=bucket, origin_access_identity=cloudfront_oai)
53 | source_config = cloudfront.SourceConfiguration(s3_origin_source=s3_origin_source, behaviors=[
54 | cloudfront.Behavior(is_default_behavior=True)])
55 |
56 | dist = cloudfront.CloudFrontWebDistribution(scope,
57 | service_name,
58 | origin_configs=[source_config],
59 | comment='CDK created',
60 | default_root_object="index.html")
61 |
62 | CfnOutput(scope, f'{service_name}_url', value=dist.distribution_domain_name,
63 | export_name=f'{service_name}-url')
64 |
65 | return codepipeline.Pipeline(scope, pipeline_name,
66 | pipeline_name=pipeline_name,
67 | stages=[
68 | codepipeline.StageProps(stage_name="Source",
69 | actions=[
70 | codepipeline_actions.CodeCommitSourceAction(
71 | action_name="CodeCommit_Source",
72 | branch="main",
73 | repository=code_commit,
74 | output=source_output,
75 | trigger=codepipeline_actions.CodeCommitTrigger.NONE)]),
76 | codepipeline.StageProps(stage_name="Build",
77 | actions=[
78 | codepipeline_actions.CodeBuildAction(
79 | action_name="SelectServiceArtifact",
80 | project=select_artifact_build,
81 | input=source_output,
82 | outputs=[service_artifact])]),
83 | codepipeline.StageProps(stage_name="Deploy",
84 | actions=[codepipeline_actions.S3DeployAction(action_name="DeployS3", bucket=bucket, input=service_artifact)]), ])
85 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # CodeCommit monorepo multi pipeline triggers
2 |
3 | This solution allows a mono-repository, which is composed of multiple services, have different CI/CD pipelines for each service. The solution detects which top level directory the modification happened and triggers the AWS CodePipeline configured to that directory.
4 |
5 | 
6 |
7 | ## Project structure
8 |
9 | This repository contains a AWS CDK project with two stacks: `MonoRepoStack` and `PipelinesStack`.
10 | `MonoRepoStack` is responsible for creating the AWS CodeCommit monorepo and the AWS Lambda with the logic to trigger the different pipelines. It is core part of the solution and doesn't need to be modified by the stack's user.
11 | `PipelinesStack` is the stack where the users are going to define their pipeline infrastructure. This repository comes with a demo and hotsite pipelines that deploys two static websites using S3 and CloudFront.
12 |
13 | 
14 |
15 | ### Directories and files
16 |
17 | #### *core*:
18 | It contains the CDK Stack that creates the CodeCommit monorepo, AWS Lambda and all the logic needed to start a specific AWS CodePipeline according to a modified file path inside the monorepo. This directory is designed to not be modified. CodeCommit is going to be created with a sample project structure defined in `monorepo-sample` directory.
19 |
20 | #### *monorepo-sample*:
21 | It contains a project structure that is going to be committed to CodeCommit after it is created. It comes with two simple static websites inside the directory.
22 |
23 | #### *pipelines*:
24 | It contains the files with the definition of each service pipeline.
25 | There is two examples inside this direcctory: `pipeline_demo.py` and `pipeline_hotsite.py`, wich demonstrates how a pipeline should be defined. It deploys the `demo` and `hotsite` websites inside the `monorepo-sample`.
26 |
27 | Stack's users are going to do their work inside of this directory, creating new python classes child of `ServicePipeline` class and implement the abstract methods: `pipeline_name()` and `build_pipeline()`.
28 |
29 | #### *monorepo_config.py*:
30 |
31 | This file contains the link between each microservice directory and its pipeline definition:
32 |
33 | ```python
34 | service_map: Dict[str, ServicePipeline] = {
35 | # folder-name -> pipeline-class
36 | 'demo': DemoPipeline(),
37 | 'hotsite': HotsitePipeline()
38 | }
39 | ```
40 |
41 | ## Running the project
42 |
43 | ### Requirements
44 |
45 | To use this solution, you need the following:
46 | * [Python 3](https://www.python.org/downloads/)
47 | * pip (package installer for Python), which already comes with Python 3 distribution: [https://pip.pypa.io/en/stable/](https://pip.pypa.io/en/stable/)
48 | * [AWS CDK](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html)
49 |
50 |
51 | ### Workspace setup
52 |
53 | 1. Clone this repository
54 | 2. Create virtualenv and install dependencies:
55 | * `make install`
56 | 3. Bootstrap the account and region for CDK:
57 | * `make bootstrap account-id= region=`
58 |
59 | ### Deploy MonoRepoStack
60 |
61 | Since this MonoRepoStack won't be modified, it should be deployed once.
62 | This stack accepts the following parameter:
63 |
64 | |Parameter name|Default Value|Type|
65 | |---|---|---|
66 | |MonorepoName| monorepo-sample | String|
67 |
68 | To deploy the stack with default parameter values, type the following command:
69 | `make deploy-core`
70 |
71 | To deploy the stack with custom parameters, type the following command:
72 | `make deploy-core monorepo-name=`
73 |
74 | You can confirm whether the resources were correctly created by getting information about the monorepo codecommit repository:
75 |
76 | `aws codecommit get-repository --repository-name `
77 |
78 | > This stack creates the AWS CodeCommit where your monorepo is going to be stored. Therefore, don't run `cdk destroy MonoRepoStack` after you have started to push modifications into, otherwise you are going to loose your remote repository.
79 |
80 | ### Deploy PipelinesStack
81 |
82 | This stack must be deployed after MonoRepoStack has already been deployed. It is going to grow accordingly to new microservices added into the monorepo code base.
83 | To deploy it, type the following command:
84 |
85 | `make deploy-pipelines`
86 |
87 | PipelinesStacks deployment prints the services URLs at the end of its execution:
88 |
89 | ```bash
90 | Outputs:
91 | PipelinesStack.demourl = .cloudfront.net
92 | PipelinesStack.hotsiteurl = .cloudfront.net
93 | ```
94 |
95 | You can check the Demo and Hotsite services working by accessing the above URLs.
96 |
97 | ### Deploy both stacks
98 |
99 | If you want to deploy both stacks on the same time, you can execute the following command for executing the deploy:
100 |
101 | `make deploy monorepo-name=`
102 |
103 | ### Checkout monorepo
104 |
105 | If you dont have a connection with the CodeCommit in your account yet, please follow the instructions of the offical documentation [here](https://docs.aws.amazon.com/codecommit/latest/userguide/how-to-connect.html).
106 |
107 | ## Steps for creating a new service pipeline
108 |
109 | In order to create a new pipeline for a new service (my-service), you should follow the bellow steps:
110 |
111 | 1. Create a new python file inside the pipelines directory: `myservice_pipeline.py`
112 | 2. Create a class `MyServicePipeline` and declare `ServicePipeline` as its super class.
113 | ```python
114 | class MyServicePipeline(ServicePipeline):
115 | def pipeline_name():
116 | return 'my-service-pipeline'
117 |
118 | def build_pipeline():
119 | # Define your CodePipeline here
120 | ```
121 | 3. In `monorepo_config.py`, import myservice_pipeline module:
122 | ```python
123 | from pipelines.myservice_pipeline import MyServicePipeline
124 | ```
125 | 4. Modify `service_map` variable inside the `monorepo_config.py` file and add the map folder -> Service Class:
126 | ```python
127 | {
128 | # ... other mappings
129 | 'my-service': MyServicePipeline()
130 | }
131 | ```
132 | 5. Redeploy PipelinesStack:
133 | ```bash
134 | make deploy-pipelines
135 | ```
136 | 6. Inside your monorepo, edit the json file `monorepo-main.json` and add the new mapping:
137 | ```js
138 | {
139 | // ... other mappings
140 | "my-service": "my-service-pipeline"
141 | }
142 | ```
143 | 7. Inside your monorepo, include the source code for the new service under a folder named `my-service`.
144 | 8. Commit and push the modification made in steps 6 and 7.
145 |
146 | ## Cleanup
147 |
148 | For deleting your stacks, execute the following command:
149 |
150 | `make destroy`
151 |
152 | After executing the command, on the Amazon S3 console, delete the buckets associated with the your pipelines starting with the following name: `pipelinesstack-codepipeline*`.
153 |
154 | > To delete a bucket, first you have to empty it and then delete it.
155 |
156 | ## Security
157 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information.
158 |
159 | ## License
160 |
161 | This library is licensed under the MIT-0 License. See the [LICENSE](LICENSE) file.
162 |
163 |
--------------------------------------------------------------------------------
/core/lambda/handler.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import base64
3 | import boto3
4 | import botocore
5 | from functools import reduce
6 | import json
7 |
8 | logger = logging.getLogger(__name__)
9 | logger.setLevel(logging.INFO)
10 |
11 | codecommit = boto3.client('codecommit')
12 | ssm = boto3.client('ssm')
13 |
14 |
15 | def main(event, context):
16 | """
17 | This AWS Lambda is triggered by AWS CodeCommit event.
18 | It starts AWS CodePipelines according to modifications in toplevel folders of the monorepo.
19 | Each toplevel folder can be associate with a different AWS CodePipeline.
20 | Must be run without concurrency - once at time for each monorepo (to avoid race condition while updating last commit parameter in SSM ParameterStore)
21 | """
22 |
23 | logger.info('event: %s', event)
24 | commit_id = get_commit_id(event)
25 | branch_name = get_branch_name(event)
26 |
27 | # logger.info('references: %s', references)
28 |
29 | repository = event['Records'][0]['eventSourceARN'].split(':')[5]
30 |
31 | paths = get_modified_files_since_last_run(
32 | repositoryName=repository, afterCommitSpecifier=commit_id, branch_name=branch_name)
33 |
34 | logger.info('paths: %s', paths)
35 | print(paths)
36 | toplevel_dirs = get_unique_toplevel_dirs(paths)
37 | print("unique toplevl dirs:", toplevel_dirs)
38 | pipeline_names = resolve_pipeline_names(toplevel_dirs, repository, branch_name)
39 | print("pipeline_names:", pipeline_names)
40 | print(start_codepipelines(pipeline_names))
41 |
42 | update_last_commit(repository, commit_id, branch_name)
43 |
44 |
45 | def get_commit_id(event):
46 | return event['Records'][0]['codecommit']['references'][0]['commit']
47 |
48 |
49 | def get_branch_name(event):
50 | branch_ref = event['Records'][0]['codecommit']['references'][0]['ref']
51 | return branch_ref.split('/')[-1]
52 |
53 |
54 | def resolve_pipeline_names(toplevel_dirs, repository, branch_name):
55 | """
56 | Look up for pipeline names according to the toplevel dir names.
57 | File name with the mapping (folder -> codepipeline-name) must be in the root level of the repo.
58 | Returns CodePipeline names that need to be triggered.
59 | """
60 | pipeline_map = codecommit.get_file(repositoryName=repository,
61 | commitSpecifier=f'refs/heads/{branch_name}', filePath=f'monorepo-{branch_name}.json')['fileContent']
62 | pipeline_map = json.loads(pipeline_map)
63 | pipeline_names = []
64 | for dir in toplevel_dirs:
65 | if dir in pipeline_map:
66 | pipeline_names.append(pipeline_map[dir])
67 | return pipeline_names
68 |
69 |
70 | def get_unique_toplevel_dirs(modified_files):
71 | """
72 | Returns toplevel folders that were modified by the last commit(s)
73 | """
74 | toplevel_dirs = set(
75 | [splitted[0] for splitted in (file.split('/') for file in modified_files)
76 | if len(splitted) > 1]
77 | )
78 |
79 | logger.info('toplevel dirs: %s', toplevel_dirs)
80 | return toplevel_dirs
81 |
82 |
83 | def start_codepipelines(codepipeline_names: list) -> dict:
84 | """
85 | start CodePipeline (s)
86 | Returns a tupple with 2 list: (success_started_pipelines, failed_to_start_pipelines)
87 | """
88 | codepipeline_client = boto3.Session().client('codepipeline')
89 |
90 | failed_codepipelines = []
91 | started_codepipelines = []
92 | for codepipeline_name in codepipeline_names:
93 | try:
94 | codepipeline_client.start_pipeline_execution(
95 | name=codepipeline_name
96 | )
97 | logger.info(f'Started CodePipeline {codepipeline_name}.')
98 | started_codepipelines.append(codepipeline_name)
99 | except codepipeline_client.exceptions.PipelineNotFoundException:
100 | logger.info(f'Could not find CodePipeline {codepipeline_name}.')
101 | failed_codepipelines.append(codepipeline_name)
102 |
103 | return (started_codepipelines, failed_codepipelines)
104 |
105 |
106 | def build_parameter_name(repository, branch_name):
107 | """
108 | Create the name of SSM ParameterStore LastCommit
109 | """
110 | # TODO must have the branch name in the parameter?
111 | return f'/MonoRepoTrigger/{repository}/{branch_name}/LastCommit'
112 |
113 |
114 | def get_last_commit(repository, commit_id, branch_name):
115 | """
116 | Get last triggered commit id.
117 | Strategy: try to find the last commit id in SSM Parameter Store '/MonoRepoTrigger/{repository}/LastCommit',
118 | if does not exist, get the parent commit from the commit that triggers this lambda
119 | Return last triggered commit hash
120 | """
121 | param_name = build_parameter_name(repository, branch_name)
122 | try:
123 | return ssm.get_parameter(Name=param_name)['Parameter']['Value']
124 | except botocore.exceptions.ClientError:
125 | logger.info('not found ssm parameter %s', param_name)
126 | commit = codecommit.get_commit(
127 | repositoryName=repository, commitId=commit_id)['commit']
128 | parent = None
129 | if commit['parents']:
130 | parent = commit['parents'][0]
131 | return parent
132 |
133 |
134 | def update_last_commit(repository, commit_id, branch_name):
135 | """
136 | Update '/MonoRepoTrigger/{repository}/LastCommit' SSM Parameter Store with the current commit that triggered the lambda
137 | """
138 | ssm.put_parameter(Name=build_parameter_name(repository, branch_name),
139 | Description='Keep track of the last commit already triggered',
140 | Value=commit_id,
141 | Type='String',
142 | Overwrite=True)
143 |
144 |
145 | def get_modified_files_since_last_run(repositoryName, afterCommitSpecifier, branch_name):
146 | """
147 | Get all modified files since last time the lambda was triggered. Developer can push several commit at once,
148 | so the number of commits between beforeCommit and afterCommit can be greater than one.
149 | """
150 |
151 | last_commit = get_last_commit(repositoryName, afterCommitSpecifier, branch_name)
152 | print("last_commit: ", last_commit)
153 | print("commit_id: ", afterCommitSpecifier)
154 | # TODO working with next_token to paginate
155 | diff = None
156 | if last_commit:
157 | diff = codecommit.get_differences(repositoryName=repositoryName, beforeCommitSpecifier=last_commit,
158 | afterCommitSpecifier=afterCommitSpecifier)['differences']
159 | else:
160 | diff = codecommit.get_differences(repositoryName=repositoryName,
161 | afterCommitSpecifier=afterCommitSpecifier)['differences']
162 |
163 | logger.info('diff: %s', diff)
164 |
165 | before_blob_paths = {d.get('beforeBlob', {}).get('path') for d in diff}
166 | after_blob_paths = {d.get('afterBlob', {}).get('path') for d in diff}
167 |
168 | all_modifications = before_blob_paths.union(after_blob_paths)
169 | return filter(lambda f: f is not None, all_modifications)
170 |
171 |
172 | if __name__ == '__main__':
173 | main({'Records': [{'codecommit': {'references': [{'commit': 'a6528d2dd877288e7c0ebdf9860d356e6d4bd073',
174 | }]}, 'eventSourceARN': ':::::repo-test-trigger-lambda'}]}, {})
175 |
--------------------------------------------------------------------------------