├── .devcontainer └── devcontainer.json ├── .github ├── CODE_OF_CONDUCT.md ├── DEVELOPERS.md ├── SECURITY.md ├── dependabot.yml └── workflows │ ├── lint-and-test.yml │ └── package-cnc.yml ├── .gitignore ├── Dockerfile ├── Dockerfile.docs ├── LICENSE.txt ├── README.md ├── cncbuilder ├── Dockerfile ├── Dockerfile.base ├── Dockerfile.dev └── README.md ├── development.md ├── docs ├── cli.md ├── configuration │ ├── cnc.md │ ├── environment_variables.md │ ├── environments.md │ ├── overview.md │ └── use_existing_vpc.md ├── customization │ ├── aws_ecs.md │ ├── deploy.md │ ├── infra_state.md │ ├── overview.md │ └── provision.md ├── examples │ ├── django-gke.md │ ├── django.md │ ├── fastapi.md │ ├── github-actions-workflow.md │ └── nextjs.md ├── fonts │ ├── TWKLausanne-0400Italic.otf │ ├── TWKLausanne-100.woff │ ├── TWKLausanne-100.woff2 │ ├── TWKLausanne-1000.woff │ ├── TWKLausanne-1000.woff2 │ ├── TWKLausanne-1000Italic.woff │ ├── TWKLausanne-1000Italic.woff2 │ ├── TWKLausanne-100Italic.woff │ ├── TWKLausanne-100Italic.woff2 │ ├── TWKLausanne-200.woff │ ├── TWKLausanne-200.woff2 │ ├── TWKLausanne-200Italic.woff │ ├── TWKLausanne-200Italic.woff2 │ ├── TWKLausanne-300.ttf │ ├── TWKLausanne-300.woff │ ├── TWKLausanne-300.woff2 │ ├── TWKLausanne-300Italic.woff2 │ ├── TWKLausanne-400.woff │ ├── TWKLausanne-400.woff2 │ ├── TWKLausanne-400Italic.otf │ ├── TWKLausanne-400Italic.ttf │ ├── TWKLausanne-400Italic.woff │ ├── TWKLausanne-400Italic.woff2 │ ├── TWKLausanne-500.ttf │ ├── TWKLausanne-500.woff │ ├── TWKLausanne-500.woff2 │ ├── TWKLausanne-500Italic.woff │ ├── TWKLausanne-600Italic.woff │ ├── TWKLausanne-600Italic.woff2 │ ├── TWKLausanne-700.woff │ ├── TWKLausanne-700.woff2 │ ├── TWKLausanne-700Italic.woff │ ├── TWKLausanne-700Italic.woff2 │ ├── TWKLausanne-800.woff │ ├── TWKLausanne-800.woff2 │ ├── TWKLausanne-900.woff2 │ ├── TWKLausanne-900Italic.woff │ └── TWKLausanne-900Italic.woff2 ├── getting-started.md ├── images │ ├── aws-infra.png │ ├── cnc_diagram_dark.png │ ├── cnc_diagram_light.png │ ├── cnc_logo_black.svg │ ├── cnc_logo_white.svg │ └── hello_world.gif ├── index.md ├── reference-architectures │ ├── aws │ │ ├── ecs.md │ │ └── lambda-lite.md │ ├── gcp │ │ ├── gke.md │ │ ├── run-lite.md │ │ └── run.md │ └── overview.md ├── stylesheets │ └── custom.css └── toolboxes.md ├── lint.sh ├── mkdocs.yml ├── nginx.conf ├── overrides ├── main.html └── partials │ └── header.html ├── poetry.lock ├── pyproject.toml ├── pytest.ini ├── requirements-docs-site.txt ├── setup.py └── src └── cnc ├── __init__.py ├── check_dependencies.py ├── commands ├── __init__.py ├── build.py ├── deploy.py ├── info.py ├── provision.py ├── render.py ├── shell.py ├── telemetry.py ├── template_editor │ ├── __init__.py │ └── inspector.py ├── toolbox.py └── update.py ├── constants.py ├── flavors ├── aws │ ├── ecs │ │ └── 1 │ │ │ ├── build │ │ │ ├── build_functions.sh.j2 │ │ │ ├── build_lambda_functions.sh.j2 │ │ │ └── main.sh.j2 │ │ │ ├── deploy │ │ │ ├── base.sh.j2 │ │ │ ├── deploy_functions.sh.j2 │ │ │ ├── deploy_lambda_functions.sh.j2 │ │ │ ├── ecs_web_task.json.j2 │ │ │ ├── env_vars.json.j2 │ │ │ ├── main.sh.j2 │ │ │ ├── partials │ │ │ │ ├── backend_deploy.sh.j2 │ │ │ │ ├── backend_deploy_status_check.sh.j2 │ │ │ │ ├── ecs_wait_with_retries.sh.j2 │ │ │ │ └── update_task_definition.sh.j2 │ │ │ ├── pre_deploy.sh.j2 │ │ │ └── pre_deploy_functions.sh.j2 │ │ │ └── provision │ │ │ ├── base.tf.j2 │ │ │ ├── frontend_routing_lambda │ │ │ └── index.js │ │ │ ├── main.tf.j2 │ │ │ └── partials │ │ │ ├── collection │ │ │ ├── has_active_backend_deployments.tf.j2 │ │ │ ├── has_active_deployments.tf.j2 │ │ │ ├── has_active_frontend_deployments.tf.j2 │ │ │ ├── has_backend_services.tf.j2 │ │ │ ├── lambda │ │ │ │ └── config_alb.tf.j2 │ │ │ └── vpc_configuration.tf.j2 │ │ │ ├── collection_level_resources.tf.j2 │ │ │ ├── environment │ │ │ ├── efs.tf.j2 │ │ │ └── msg_queue_and_obj_storage.tf.j2 │ │ │ ├── environment_level_resources.tf.j2 │ │ │ ├── service │ │ │ ├── active_backend.tf.j2 │ │ │ ├── active_frontend.tf.j2 │ │ │ └── backend │ │ │ │ ├── alb_target_group.tf.j2 │ │ │ │ ├── ecs_service.tf.j2 │ │ │ │ ├── scheduled_tasks.tf.j2 │ │ │ │ └── workers.tf.j2 │ │ │ └── service_level_resources.tf.j2 │ ├── lambda-lite │ │ └── 1 │ │ │ ├── build │ │ │ ├── build_functions.sh.j2 │ │ │ └── main.sh.j2 │ │ │ ├── deploy │ │ │ ├── base.sh.j2 │ │ │ ├── deploy_functions.sh.j2 │ │ │ ├── env_vars.json.j2 │ │ │ └── main.sh.j2 │ │ │ └── provision │ │ │ ├── base.tf.j2 │ │ │ ├── main.tf.j2 │ │ │ └── partials │ │ │ └── collection │ │ │ └── environments.tf.j2 │ └── shared │ │ ├── .metadata │ │ ├── build │ │ └── env_vars.sh.j2 │ │ ├── deploy │ │ └── deploy_functions_base.sh.j2 │ │ ├── provision │ │ ├── lambda_function_payload │ │ │ └── lambda_function.py │ │ └── partials │ │ │ └── collection │ │ │ ├── database │ │ │ └── rds.tf.j2 │ │ │ ├── dynamodb │ │ │ ├── dynamodb.tf.j2 │ │ │ ├── outputs.tf.j2 │ │ │ └── variables.tf.j2 │ │ │ ├── lambda │ │ │ ├── lambda.tf.j2 │ │ │ ├── outputs.tf.j2 │ │ │ └── variables.tf.j2 │ │ │ └── providers.tf.j2 │ │ └── toolbox │ │ ├── Dockerfile.j2 │ │ ├── main.sh.j2 │ │ └── partials │ │ ├── proxy_only.sh.j2 │ │ └── service_container.sh.j2 └── gcp │ ├── gke │ └── 1 │ │ ├── .metadata │ │ ├── build │ │ └── main.sh.j2 │ │ ├── deploy │ │ ├── base.sh.j2 │ │ ├── deploy.sh.j2 │ │ ├── deploy_functions.sh.j2 │ │ ├── k8s │ │ │ └── deployment.yml.j2 │ │ └── main.sh.j2 │ │ └── provision │ │ ├── base.tf.j2 │ │ ├── main.tf.j2 │ │ └── partials │ │ ├── collection_level_resources.tf.j2 │ │ └── service_level_resources.tf.j2 │ ├── run-lite │ └── 1 │ │ ├── build │ │ └── main.sh.j2 │ │ ├── deploy │ │ ├── base.sh.j2 │ │ ├── deploy_functions.sh.j2 │ │ ├── main.sh.j2 │ │ └── run-container.yml.j2 │ │ ├── provision │ │ ├── base.tf.j2 │ │ └── main.tf.j2 │ │ └── toolbox │ │ └── main.sh.j2 │ ├── run │ └── 1 │ │ ├── build │ │ └── main.sh.j2 │ │ ├── cnc-flavor.yml │ │ ├── deploy │ │ ├── base.sh.j2 │ │ ├── deploy_functions.sh.j2 │ │ └── main.sh.j2 │ │ └── provision │ │ ├── base.tf.j2 │ │ └── main.tf.j2 │ └── shared │ ├── .metadata │ ├── build │ ├── Dockerfile.run.j2 │ ├── base.sh.j2 │ ├── build_functions.sh.j2 │ ├── env_vars.sh.j2 │ └── nginx.conf.j2 │ ├── deploy │ ├── all-access-policy.yml.j2 │ ├── deploy_functions_base.sh.j2 │ ├── k8s │ │ ├── scheduled_tasks.yml │ │ ├── service_containers │ │ │ └── backend-worker.yml │ │ └── workers.yml │ ├── run-container.yml.j2 │ ├── run-job.yml.j2 │ └── run-svc.yml.j2 │ ├── provision │ └── partials │ │ ├── collection │ │ ├── bastion_instance.tf.j2 │ │ ├── common_setup_resources.tf.j2 │ │ ├── default_service.tf.j2 │ │ ├── gke_cluster.tf.j2 │ │ ├── providers.tf.j2 │ │ └── vpc_networking.tf.j2 │ │ ├── environment │ │ ├── database_resources.tf.j2 │ │ ├── object_storage_resources.tf.j2 │ │ └── resource_managed_secrets.tf.j2 │ │ └── service │ │ ├── cache.tf.j2 │ │ ├── cloud_run_service.tf.j2 │ │ ├── cloud_run_service_networking.tf.j2 │ │ └── kubernetes_workloads.tf.j2 │ └── toolbox │ ├── Dockerfile.j2 │ ├── main.sh.j2 │ └── partials │ ├── proxy_only.sh.j2 │ └── service_container.sh.j2 ├── logger.py ├── main.py ├── models ├── __init__.py ├── application.py ├── base_model.py ├── builder.py ├── config │ ├── __init__.py │ ├── config.py │ ├── deploy_resource_limits.py │ ├── resource.py │ ├── service.py │ ├── settings.py │ └── utils.py ├── custom_header.py ├── cycle_stage_base.py ├── deployer.py ├── environment.py ├── environment_collection.py ├── environment_variable.py ├── providers │ ├── __init__.py │ ├── amazon │ │ ├── __init__.py │ │ ├── cache_resource_settings.py │ │ ├── custom_headers.py │ │ ├── database_resource_settings.py │ │ ├── deploy_resource_limits.py │ │ ├── environment_collection.py │ │ └── service_account.py │ └── google │ │ ├── __init__.py │ │ ├── cache_resource_settings.py │ │ ├── database_resource_settings.py │ │ ├── deploy_resource_limits.py │ │ ├── environment_collection.py │ │ └── service_account.py ├── provisioner.py ├── resource_use_existing.py ├── stage.py └── toolbox.py ├── tests ├── __init__.py ├── base_test_class.py ├── fixtures │ ├── backend-1-serverless-1-service-1-db-1-dynamo-1 │ │ ├── cnc.yml │ │ └── environments_backend_1_serverless_1_service_1_db_1_dynamo.yml │ ├── backend-1-serverless-1-service-1-db │ │ ├── cnc.yml │ │ └── environments_backend_1_serverless_1_service_1_db.yml │ ├── backend-1-serverless-1-service-2-db │ │ ├── cnc.yml │ │ └── environments_backend_1_serverless_1_service_2_db.yml │ ├── backend-1-serverless-2-service-2-db-2-dynamo-2 │ │ ├── cnc.yml │ │ └── environments_backend_1_serverless_2_service_2_db_2_dynamo.yml │ ├── backend-1-service-1-db-both-use-existing │ │ ├── cnc.yml │ │ └── environments.yml │ ├── backend-1-service-1-db-use-existing │ │ ├── cnc.yml │ │ └── environments.yml │ ├── backend-1-service-1-db-variables-aliases │ │ ├── cnc.yml │ │ └── environments.yml │ ├── backend-1-service-1-db │ │ ├── cnc.yml │ │ ├── environments_aws_ecs_existing_vpc_cidrs.yml │ │ └── environments_aws_ecs_existing_vpc_subnets.yml │ ├── backend-1-service-1-worker-1-task │ │ └── cnc.yml │ ├── backend-1-service-2-db-2-envs │ │ ├── cnc.yml │ │ ├── environments.yml │ │ ├── environments_aws_ecs.yml │ │ └── environments_gcp_run_region_settings.yml │ ├── backend-1-service-2-db │ │ └── cnc.yml │ ├── backend-1-service-custom │ │ ├── cnc.yml │ │ ├── custom │ │ │ └── deploy │ │ │ │ └── main.sh.j2 │ │ └── environments.yml │ ├── backend-1-service-existing │ │ └── cnc.yml │ ├── backend-1-service-invalid │ │ └── cnc.yml │ ├── backend-1-service │ │ ├── cnc.yml │ │ └── environments_json_var.yml │ ├── backend-2-serverless-2-service-2-db-2-dynamo-2 │ │ ├── cnc.yml │ │ └── environments_backend_2_serverless_2_service_2_db_2_dynamo.yml │ ├── backend-2-serverless-2-service-2-db │ │ ├── cnc.yml │ │ └── environments_backend_2_serverless_2_service_2_db.yml │ ├── backend-2-service-1-db │ │ └── cnc.yml │ ├── backend-2-service-2-db │ │ └── cnc.yml │ ├── serverless-1-service-1-db │ │ ├── cnc.yml │ │ └── environments_serverless_1_service_1_db.yml │ ├── serverless-1-service-1-dynamodb-1-db │ │ ├── cnc.yml │ │ └── environments_serverless_1_service_1_dynamodb_1_db.yml │ ├── serverless-1-service-1-dynamodb │ │ ├── cnc.yml │ │ └── environments_serverless_1_service_1_dynamodb.yml │ ├── serverless-1-service-2-db │ │ ├── cnc.yml │ │ └── environments_serverless_1_service_2_db.yml │ ├── serverless-1-service-2-dynamodb-2-db │ │ ├── cnc.yml │ │ └── environments_serverless_1_service_2_dynamodb_2_db.yml │ ├── serverless-1-service-2-dynamodb │ │ ├── cnc.yml │ │ └── environments_serverless_1_service_2_dynamodb.yml │ ├── serverless-1-service │ │ ├── cnc.yml │ │ └── environments_serverless_1_service.yml │ ├── serverless-2-service-2-dynamodb-2-db │ │ ├── cnc.yml │ │ └── environments_serverless_2_service_2_dynamodb_2_db.yml │ ├── serverless-2-service-2-dynamodb │ │ ├── cnc.yml │ │ └── environments_serverless_2_service_2_dynamodb.yml │ └── shared │ │ ├── cnc.yml │ │ ├── environments-custom-provision.yml │ │ ├── environments.yml │ │ ├── environments_aws_ecs.yml │ │ ├── environments_aws_ecs_existing_vpc.yml │ │ ├── environments_gcp_run_lite.yml │ │ └── environments_weird_character_vars.yml ├── test_builder.py ├── test_deployer.py ├── test_environment_collection.py ├── test_environment_variables.py ├── test_environments.py ├── test_provisioner.py ├── test_services.py └── test_toolbox.py └── utils ├── __init__.py └── string.py /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | // README at: https://github.com/devcontainers/templates/tree/main/src/python 3 | { 4 | "name": "CNC dev container", 5 | "build": {"dockerfile": "../Dockerfile", "context": "../"}, 6 | "features": { 7 | "ghcr.io/devcontainers/features/aws-cli:1": {}, 8 | "ghcr.io/devcontainers-contrib/features/terraform-asdf:2": {}, 9 | "ghcr.io/dhoeric/features/google-cloud-cli:1": {}, 10 | "ghcr.io/devcontainers/features/docker-outside-of-docker": {} 11 | }, 12 | 13 | // Features to add to the dev container. More info: https://containers.dev/features. 14 | // "features": {}, 15 | 16 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 17 | // "forwardPorts": [], 18 | 19 | // Use 'postCreateCommand' to run commands after the container is created. 20 | "postCreateCommand": "poetry install --with dev" 21 | 22 | // Configure tool-specific properties. 23 | // "customizations": {}, 24 | 25 | // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. 26 | // "remoteUser": "root" 27 | } 28 | -------------------------------------------------------------------------------- /.github/DEVELOPERS.md: -------------------------------------------------------------------------------- 1 | ## Getting started 2 | 3 | Thank you for your interest in [Coherence](https://withcoherence.com) and your willingness to contribute! 4 | 5 | To ensure a positive and inclusive environment, please read our [code of conduct](./CODE_OF_CONDUCT.md). We encourage you to explore the existing [issues](https://github.com/coherenceplatform/cnc/issues) to see how you can make a meaningful impact. This document will help you setup your development environment. -------------------------------------------------------------------------------- /.github/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | CNC applies security fixes as far back as reasonably possible. 6 | 7 | ## Reporting a Vulnerability 8 | 9 | We encourage responsible disclosure practices for security vulnerabilities. Please contact security@withcoherence.com to report a security concern before opening a PR or Issue which will be publicly visible. -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for more information: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | # https://containers.dev/guide/dependabot 6 | 7 | version: 2 8 | updates: 9 | - package-ecosystem: "devcontainers" 10 | directory: "/" 11 | schedule: 12 | interval: weekly 13 | -------------------------------------------------------------------------------- /.github/workflows/lint-and-test.yml: -------------------------------------------------------------------------------- 1 | name: Lint and Test 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | python-version: ["3.9", "3.10", "3.11"] 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | - name: Install Poetry 21 | run: | 22 | curl -sSL https://install.python-poetry.org | python3 - 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v3 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | cache: "poetry" 28 | - name: Install dependencies 29 | run: | 30 | poetry install --with dev 31 | - name: Test with pytest 32 | run: | 33 | poetry run pytest 34 | - name: Lint and format 35 | run: | 36 | poetry run ./lint.sh -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # CNC Team Preferences 3 | notes.* 4 | 5 | # Dev Environment files 6 | .venv 7 | __pycache__ 8 | dist/* 9 | 10 | # Managed Terraform Artifacts 11 | *terraform* -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12 2 | 3 | RUN apt-get update && apt-get install -y less \ 4 | vim jq 5 | RUN pip install poetry 6 | 7 | COPY . . 8 | 9 | RUN poetry install --with dev -------------------------------------------------------------------------------- /Dockerfile.docs: -------------------------------------------------------------------------------- 1 | FROM python:3.11 AS mkdocs-builder 2 | 3 | RUN mkdir /app 4 | WORKDIR /app 5 | 6 | COPY requirements-docs-site.txt requirements.txt 7 | RUN pip install -r requirements.txt 8 | COPY . . 9 | 10 | RUN mkdir /build 11 | RUN mkdocs build --site-dir /build 12 | 13 | FROM nginx 14 | COPY --from=mkdocs-builder /build /usr/share/nginx/html/ 15 | COPY ./nginx.conf /etc/nginx/nginx.conf -------------------------------------------------------------------------------- /cncbuilder/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM us-docker.pkg.dev/coherence-public/public/cnc_base 2 | 3 | ARG CNC_VERSION="0.1.68" 4 | 5 | RUN --mount=type=cache,target=/root/.cache/pip \ 6 | pip cache purge && \ 7 | pip install --no-cache-dir cocnc==${CNC_VERSION} 8 | -------------------------------------------------------------------------------- /cncbuilder/Dockerfile.base: -------------------------------------------------------------------------------- 1 | FROM python:3.12 2 | 3 | ENV PATH=$PATH:/root/google-cloud-sdk/bin 4 | ARG DOCKER_VERSION="5:24.0.9-1~debian.12~bookworm" 5 | 6 | WORKDIR /tmp 7 | 8 | RUN apt-get update && apt-get install -y lsb-release \ 9 | apt-transport-https ca-certificates 10 | 11 | # Docker packages 12 | RUN curl -fsSL https://download.docker.com/linux/debian/gpg | \ 13 | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ 14 | echo "deb [arch=$(dpkg --print-architecture) \ 15 | signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] \ 16 | https://download.docker.com/linux/debian \ 17 | $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list 18 | 19 | # Kubernetes packages 20 | RUN curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | \ 21 | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg && \ 22 | echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] \ 23 | https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | \ 24 | tee /etc/apt/sources.list.d/kubernetes.list 25 | 26 | # Google Cloud SDK packages 27 | RUN curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ 28 | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \ 29 | echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \ 30 | https://packages.cloud.google.com/apt cloud-sdk main" | \ 31 | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list 32 | 33 | RUN apt-get update && apt-get -y dist-upgrade && \ 34 | apt-get install -y less \ 35 | zip \ 36 | jq \ 37 | groff \ 38 | mandoc \ 39 | docker-ce=${DOCKER_VERSION} \ 40 | docker-ce-cli=${DOCKER_VERSION} \ 41 | docker-compose \ 42 | docker-compose-plugin \ 43 | kubectl \ 44 | google-cloud-sdk-gke-gcloud-auth-plugin && \ 45 | apt-get autoremove && \ 46 | apt-get clean 47 | 48 | RUN curl -sSL https://releases.hashicorp.com/terraform/1.5.7/terraform_1.5.7_linux_amd64.zip -o terraform.zip && \ 49 | unzip terraform.zip && \ 50 | mv terraform /usr/local/bin/ && \ 51 | rm terraform.zip 52 | 53 | RUN curl -sSL https://sdk.cloud.google.com | bash 54 | 55 | RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ 56 | unzip awscliv2.zip && \ 57 | ./aws/install 58 | 59 | RUN curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb" && \ 60 | dpkg -i session-manager-plugin.deb 61 | 62 | RUN curl -sSL https://nixpacks.com/install.sh | bash 63 | 64 | WORKDIR /app 65 | 66 | CMD ["/bin/bash"] 67 | -------------------------------------------------------------------------------- /cncbuilder/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM cnc_base 2 | 3 | WORKDIR /tmp/cnc 4 | 5 | COPY . . 6 | 7 | RUN pip install . 8 | 9 | WORKDIR /app 10 | -------------------------------------------------------------------------------- /cncbuilder/README.md: -------------------------------------------------------------------------------- 1 | # Building the cnc image 2 | 3 | ```bash 4 | # From the base directory of this repository 5 | 6 | # base image all others are based on 7 | docker build -t cnc_base -f cncbuilder/Dockerfile.base . 8 | 9 | # Builds "prod" image (installs cnc from pypi) 10 | docker build -t us-docker.pkg.dev/coherence-public/public/cnc -f cncbuilder/Dockerfile . 11 | 12 | # Builds "dev" image (installs cnc from local fs) 13 | docker build -t my_cnc_dev_image -f cncbuilder/Dockerfile.dev . 14 | ``` 15 | -------------------------------------------------------------------------------- /development.md: -------------------------------------------------------------------------------- 1 | Contributions via PR to this repo are welcome. Please fork the repo, work on your feature or change using the guide below, and then make a PR against this repo. The team will review the PR and work with you to get it merged. Thanks! 2 | 3 | # install pipx 4 | [pipx installation instructions](https://pipx.pypa.io/stable/installation) 5 | 6 | e.g. for macOS 7 | ```bash 8 | brew install pipx 9 | pipx ensurepath 10 | ``` 11 | 12 | # install poetry 13 | [poetry](https://python-poetry.org/docs/) 14 | 15 | e.g. 16 | ```bash 17 | pipx install poetry 18 | ``` 19 | 20 | # install dev dependencies 21 | ```bash 22 | poetry install --with dev 23 | ``` 24 | Poetry will also install a script so the cli 25 | can be run using `cnc` as if there was a binary installed. 26 | 27 | 28 | # start a shell in the python virtual env 29 | ```bash 30 | # starts a login shell in the virtual env created by poetry 31 | # pre-requisite for the following commands (python shell, cli, etc.) 32 | poetry shell 33 | ``` 34 | 35 | # running the tests 36 | Run this in the `src` dir after running `poetry shell` as per above. 37 | 38 | ```bash 39 | pytest 40 | ``` 41 | 42 | You can also run something like this to run one class (no need to qualify with path): 43 | ```pytest -k EnvironmentCollectionExistingDBTestCase -v``` 44 | 45 | # start an interactive python shell (ipython) 46 | You can do this from anywhere once you have activated poetry shell 47 | 48 | ```bash 49 | cnc shell start 50 | ``` 51 | e.g.: 52 | - start in your `cnc` repo 53 | - `poetry shell` 54 | - `cd ../MY_SITE_DIR` 55 | - `cnc shell start` 56 | 57 | 58 | # running the cli 59 | ```bash 60 | cnc --help 61 | ``` 62 | 63 | # dependencies 64 | 65 | ## Python 66 | developed using python 🐍 >= 3.9 67 | (check yours with `python --version`) 68 | 69 | There's a codespace set up in the repo where you just run `poetry shell` and you're all set to work, that's the easiest place to be. 70 | 71 | 72 | # Releasing a new version 73 | 74 | ## Setup PyPI creds 75 | 76 | e.g. populate ~/.pypirc 77 | 78 | ## Steps 79 | 80 | - `python3 -m pip install --upgrade build` 81 | - `python3 -m build` 82 | - `python3 -m twine upload dist/*` -------------------------------------------------------------------------------- /docs/configuration/environment_variables.md: -------------------------------------------------------------------------------- 1 | # Environment Variables Info 2 | 3 | Variables are defined in `environments.yml`. Each environment defines its own variables. You can use [YAML Anchors](https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/) to re-use the same variables across many environments. 4 | 5 | ## Types 6 | 7 | - `standard` where the `value` is defined 8 | - `secret` where the `secret_id` is provided. `cnc` will not create this secret, you create it in your cloud and `cnc` will populate appropriately in the app's lifecycle 9 | - `output` where a `terraform` output value is referenced. See the bottom for useful info on output name templating. 10 | - `alias` where a value references another variable's value. This can be useful if and application requires multiple values or when migrating between 2 values. It's especially useful for `cnc`-managed variables 11 | 12 | ## CNC Managed 13 | 14 | Certain values such as environment name are automatically provided by `cnc`. These are: 15 | 16 | - `CNC_APPLICATION_NAME` 17 | - `CNC_ENVIRONMENT_NAME` 18 | - `CNC_ENVIRONMENT_DOMAIN` 19 | 20 | Additionally, if resources such as database are configured, you'll see variables: 21 | 22 | - `DB_PASSWORD` 23 | - `DATABASE_URL` 24 | - `REDIS_URL` 25 | - `...` more, check the code for now to see them all... 26 | 27 | # Example 28 | 29 | ```yaml 30 | environments: 31 | - name: dev 32 | 33 | # cnc also adds "managed" variables e.g. domain/environment/resource info 34 | environment_variables: 35 | # see the environment variable docs for more 36 | - name: FOO 37 | value: bar 38 | 39 | # this will reference a secret ID in the cloud 40 | # cnc will not create this secret, you create it 41 | # see flavor docs for secret format 42 | - name: FOO_SECRET 43 | secret_id: bar123 44 | 45 | # reference any terraform output 46 | # see flavor docs for available outputs 47 | - name: FOO_OUTPUT_1 48 | output_name: bar123-a 49 | 50 | # alias another variable name add another copy with a new name 51 | # you can alias any variable type 52 | - name: FOO_OTHER_ALIAS_NAME 53 | alias: foo-standard 54 | ``` 55 | 56 | 57 | # Output Name Templating 58 | 59 | You can use `environment.name` and `collection.name` in the output names you reference, for example: 60 | 61 | ``` 62 | - name: foo-output 63 | output_name: "{{ environment.name }}-a" 64 | ``` 65 | 66 | For an environment named `dev` this will result in: `dev-a` as the `output` which `cnc` will pull from `terraform`. -------------------------------------------------------------------------------- /docs/configuration/environments.md: -------------------------------------------------------------------------------- 1 | # The CNC environments file 2 | 3 | `cnc` uses a `yml` file to define environments that it should provision, build, and deploy. Here's a commented example: 4 | 5 | ```yaml 6 | # name of your app 7 | name: my-first-app 8 | 9 | # which cloud provider? gcp or aws 10 | provider: gcp 11 | 12 | # OPTIONAL, can define on each collection instead 13 | region: us-east1 14 | 15 | # which flavor 16 | # see the flavors docs 17 | # you can add your own flavor in your repo as well 18 | flavor: run-light 19 | 20 | # version of the flavor 21 | version: 1 22 | 23 | # OPTIONAL: used if customizing 24 | # see customization docs for more info 25 | template_config: 26 | # from root, where do we look for custom templates? 27 | template_directory: custom 28 | # OPTIONAL, defaults to main.tf.j2 29 | provision_filename: "mymaintf.tf.j2" 30 | # OPTIONAL, defaults to main.sh.j2 31 | deploy_filename: "mydeployscript.sh.j2" 32 | # OPTIONAL, defaults to main.sh.j2 33 | build_filename: "mybuildscript.sh.j2" 34 | 35 | # you can have many collections 36 | collections: 37 | # name of the collection 38 | - name: dev 39 | # each environment in the collection will get a subdomain of this base_domain 40 | # OPTIONAL: will default to cloud-provided URL if not defined 41 | base_domain: mydevsite.com 42 | 43 | # OPTIONAL, uses app default if not provided 44 | region: us-east1 45 | 46 | # each collection can live in a different cloud account 47 | # 48 | account_id: "foo-bar-123" 49 | environments: 50 | - name: dev 51 | 52 | # cnc also adds "managed" variables e.g. domain/environment/resource info 53 | environment_variables: 54 | 55 | # see the environment variable docs for more 56 | - name: FOO 57 | value: bar 58 | 59 | # this will reference a secret ID in the cloud 60 | # cnc will not create this secret, you create it 61 | # see flavor docs for secret format 62 | - name: foo-secret 63 | secret_id: bar123 64 | 65 | # reference any terraform output 66 | # see flavor docs for available outputs 67 | - name: foo-output 68 | output_name: bar123-a 69 | 70 | # alias another variable name add another copy with a new name 71 | # you can alias any variable type 72 | - name: foo-standard-alias 73 | alias: foo-standard 74 | ``` -------------------------------------------------------------------------------- /docs/configuration/overview.md: -------------------------------------------------------------------------------- 1 | # CNC Configuration 2 | 3 | - [cnc.yml](./cnc.md) 4 | - [environments.yml](./environments.md) 5 | 6 | ## Environment Variables 7 | 8 | Managing environment variables is one of the most common interactions with environments. `cnc` makes it easy, read more [here](./environment_variables.md). 9 | 10 | ## Using Existing Resources 11 | 12 | - VPC: [aws](./use_existing_vpc.md) -------------------------------------------------------------------------------- /docs/customization/deploy.md: -------------------------------------------------------------------------------- 1 | # Customizing deploy for your app 2 | 3 | Let's say that you want to run some additional commands before or after the included deploy functions for each service. This enables you to do things like configure integrations or other systems to be ready to work with your new service. You could also replace the deployment logic entirely. 4 | 5 | ## environment.yml custom templates config 6 | 7 | Add this to `environments.yml`: 8 | 9 | ```yaml 10 | template_config: 11 | # from root, where do we look for custom templates? 12 | template_directory: custom 13 | ``` 14 | 15 | ## custom template creation 16 | 17 | Let's say you want to add a command to run after your docker build and push steps included in the default. 18 | 19 | Add `main.sh.j2` into the `deploy` folder in the directory you set as the `template_directoy` in your project. 20 | 21 | ```sh 22 | {% extends "base.sh.j2" %} 23 | 24 | {% block build_commands %} 25 | # this is a jinja function that calls the block you're inheriting from 26 | # you can leave this out if you don't want to default resources for this block 27 | {{ super() }} 28 | echo "hello {{ environment.name }}" 29 | {% endblock %} 30 | ``` 31 | 32 | You could also replace the build entirely and use alternative builders like `depot` if you wanted to. 33 | 34 | ## block options 35 | 36 | This is the default `main.sh.j2` with the available blocks. 37 | 38 | ```sh 39 | {% extends "base.sh.j2" %} 40 | 41 | {# All Available Blocks #} 42 | 43 | {# before deploy #} 44 | {# {% block install_commands %} #} 45 | {# {% endblock install_commands %} #} 46 | 47 | {# deploy body #} 48 | {# {% block build_commands %} #} 49 | {# {% endblock %} #} 50 | 51 | {# post deploy #} 52 | {# {% block finally_build_commands %} #} 53 | {# {% endblock %} #} 54 | ``` 55 | 56 | # Test your changes 57 | 58 | Run `cnc deploy perform --debug` and see your new commands in action! -------------------------------------------------------------------------------- /docs/customization/infra_state.md: -------------------------------------------------------------------------------- 1 | # TF State Customization 2 | 3 | By default, [terraform state](https://developer.hashicorp.com/terraform/language/state) will be stored in the `.terraform` directory wherever you run `cnc`. Usually this would be your project root. 4 | 5 | This is not optimal for a few reasons: 6 | - If you lost the directory, you'd lose the state, and possibly lose track of cloud resources 7 | - You cannot share the state with other members of your project with this configuration or run the `provision` step from CI/CD (you can always run `build` or `deploy` but variables that reference state will not resolve properly) 8 | 9 | If you want to change the state location, it's easy to use any state backend, just like plain `terraform`. 10 | 11 | # Customizing the state backend 12 | 13 | As with other `provision` customizations, you add this to `environments.yml`: 14 | 15 | ```yaml 16 | template_config: 17 | # from root, where do we look for custom templates? 18 | template_directory: custom 19 | ``` 20 | 21 | Put this in `main.tf.j2` in the `provision` folder in the `template_directory`. 22 | 23 | ```yaml 24 | {% extends "base.tf.j2" %} 25 | 26 | {% block state %} 27 | terraform { 28 | backend "s3" { 29 | bucket = "my-tf-state-bucket" 30 | } 31 | } 32 | {% endblock %} 33 | ``` -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-0400Italic.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-0400Italic.otf -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-100.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-100.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-100.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-100.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-1000.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-1000.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-1000.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-1000.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-1000Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-1000Italic.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-1000Italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-1000Italic.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-100Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-100Italic.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-100Italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-100Italic.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-200.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-200.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-200.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-200.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-200Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-200Italic.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-200Italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-200Italic.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-300.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-300.ttf -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-300.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-300.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-300.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-300.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-300Italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-300Italic.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-400.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-400.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-400Italic.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-400Italic.otf -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-400Italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-400Italic.ttf -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-400Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-400Italic.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-400Italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-400Italic.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-500.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-500.ttf -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-500.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-500.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-500.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-500.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-500Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-500Italic.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-600Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-600Italic.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-600Italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-600Italic.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-700.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-700.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-700.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-700.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-700Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-700Italic.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-700Italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-700Italic.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-800.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-800.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-800.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-800.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-900.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-900.woff2 -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-900Italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-900Italic.woff -------------------------------------------------------------------------------- /docs/fonts/TWKLausanne-900Italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/fonts/TWKLausanne-900Italic.woff2 -------------------------------------------------------------------------------- /docs/images/aws-infra.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/images/aws-infra.png -------------------------------------------------------------------------------- /docs/images/cnc_diagram_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/images/cnc_diagram_dark.png -------------------------------------------------------------------------------- /docs/images/cnc_diagram_light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/images/cnc_diagram_light.png -------------------------------------------------------------------------------- /docs/images/cnc_logo_black.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/images/cnc_logo_white.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/images/hello_world.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/docs/images/hello_world.gif -------------------------------------------------------------------------------- /docs/reference-architectures/overview.md: -------------------------------------------------------------------------------- 1 | # CNC Reference Architectures 2 | 3 | A `flavor` is a reference architecture built into `cnc`. It's a set of IaC (`provision`), build scripts, and deploy scripts, that work together to create a working environment. As discussed in [customization](/customization/overview/) you can also customize (in part or whole) these architectures further in your own repo, up to fully writing your own architecture that uses none of the included parts. 4 | 5 | You specify the app's flavor in `environments.yml`, e.g. 6 | 7 | ```yaml 8 | # name of your app 9 | name: my-first-app 10 | provider: gcp 11 | region: us-east1 12 | flavor: run-light 13 | ``` 14 | 15 | ## Included Architectures 16 | 17 | `run-lite` is the fastest and cheapest flavor to get started with. It is designed to be as low cost as possible and will often fall under "free forever" pricing from GCP. 18 | 19 | ### AWS 20 | 21 | - [ecs](./aws/ecs.md) 22 | - [lambda-lite](./aws/lambda-lite.md) 23 | 24 | ### GCP 25 | 26 | - [run](./gcp/run.md) 27 | - [gke](./gcp/gke.md) 28 | - [run-lite](./gcp/run-lite.md) 29 | 30 | 31 | ## Developing your own flavor 32 | 33 | You can customize `cnc` limitlessly. See [Customizing CNC](/customization/overview/) for more. 34 | -------------------------------------------------------------------------------- /lint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | black --check . 6 | ruff check --ignore F811 --exclude __init__.py . -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: CNC Framework Site 2 | site_url: https://cncframework.com 3 | repo_url: https://github.com/coherenceplatform/cnc 4 | theme: 5 | name: material 6 | custom_dir: overrides 7 | logo: images/cnc_logo_white.svg 8 | extra_css: 9 | - 'stylesheets/custom.css' -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | events { 2 | worker_connections 1024; 3 | } 4 | 5 | http { 6 | include mime.types; 7 | default_type application/octet-stream; 8 | map_hash_bucket_size 128; 9 | port_in_redirect off; 10 | 11 | map $http_host $host_without_port { 12 | ~^(.+):\d+$ $1; 13 | default $http_host; 14 | } 15 | 16 | map $request_uri $redirect_url { 17 | ~^/flavors/overview/?$ https://cncframework.com/reference-architectures/overview/; 18 | ~^/flavors/aws/ecs/?$ https://cncframework.com/reference-architectures/aws/ecs/; 19 | ~^/flavors/gcp/run/?$ https://cncframework.com/reference-architectures/gcp/run/; 20 | ~^/flavors/gcp/run-lite/?$ https://cncframework.com/reference-architectures/gcp/run-lite/; 21 | ~^/flavors/gcp/gke/?$ https://cncframework.com/reference-architectures/gcp/gke/; 22 | } 23 | 24 | server { 25 | listen 8080; 26 | server_name _; 27 | 28 | proxy_http_version 1.1; 29 | proxy_set_header Upgrade $http_upgrade; 30 | proxy_set_header Connection 'upgrade'; 31 | proxy_set_header Host $host_without_port; 32 | proxy_cache_bypass $http_upgrade; 33 | proxy_buffers 4 16k; 34 | proxy_buffer_size 8k; 35 | proxy_busy_buffers_size 16k; 36 | 37 | gzip on; 38 | gzip_proxied any; 39 | gzip_comp_level 6; 40 | gzip_buffers 16 8k; 41 | gzip_http_version 1.1; 42 | gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; 43 | 44 | if ($redirect_url) { 45 | return 301 $redirect_url; 46 | } 47 | 48 | location / { 49 | proxy_set_header Host $host_without_port; 50 | proxy_set_header X-Real-IP $remote_addr; 51 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 52 | proxy_set_header X-Forwarded-Proto $scheme; 53 | 54 | root /usr/share/nginx/html; 55 | 56 | try_files $uri $uri/ =404; 57 | } 58 | } 59 | } -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "cocnc" 3 | version = "0.1.68" 4 | description = "CNC is the first framework for application deployment, a fully distributed and customizable PaaS developer experience, based on docker-compose config files" 5 | authors = [ 6 | "Adam Abdelaziz ", 7 | "Zachary Zaro ", 8 | ] 9 | readme = "README.md" 10 | packages = [ 11 | { include = "cnc", from = "src" }, 12 | ] 13 | homepage="https://github.com/coherenceplatform/cnc" 14 | repository="https://github.com/coherenceplatform/cnc" 15 | 16 | [tool.poetry.scripts] 17 | cnc = 'cnc.main:app' 18 | 19 | [tool.poetry.dependencies] 20 | python = "^3.9" 21 | typer = {extras = ["all"], version = "^0.9.0"} 22 | pydantic = "2.6.3" 23 | pyyaml = "^6.0.1" 24 | pygohcl = "^1.0.8" 25 | jinja2 = "^3.1.3" 26 | tabulate = "^0.9.0" 27 | google-cloud-secret-manager = "^2.19.0" 28 | rudder-sdk-python = "^2.1.0" 29 | py-machineid = "^0.5.1" 30 | boto3 = "^1.34.78" 31 | ipython = "^8.0.0" 32 | toml = "^0.10.2" 33 | 34 | [tool.poetry.group.dev.dependencies] 35 | black = "^24.3.0" 36 | ruff = "^0.3.3" 37 | pytest = "^8.1.1" 38 | mkdocs-material = "^9.5.24" 39 | 40 | [build-system] 41 | requires = ["poetry-core"] 42 | build-backend = "poetry.core.masonry.api" 43 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore::DeprecationWarning -------------------------------------------------------------------------------- /requirements-docs-site.txt: -------------------------------------------------------------------------------- 1 | mkdocs-material -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | import toml 3 | import setuptools 4 | 5 | 6 | def read_pyproject(): 7 | with open("pyproject.toml", "r", encoding="utf-8") as f: 8 | return toml.load(f) 9 | 10 | 11 | def build_setup_args(): 12 | pyproject = read_pyproject() 13 | poetry_config = pyproject["tool"]["poetry"] 14 | 15 | # Map the necessary parts from the pyproject.toml to setup() arguments 16 | return { 17 | "name": poetry_config["name"], 18 | "version": "0.1", 19 | "description": poetry_config["description"], 20 | "author": ", ".join(a for a in poetry_config["authors"]), 21 | "packages": setuptools.find_packages(where="src"), 22 | "package_dir": {"": "src"}, 23 | "entry_points": {"console_scripts": ["cnc=src.main:app"]}, 24 | "install_requires": [str(v) for v in poetry_config["dependencies"].values()], 25 | "extras_require": { 26 | "dev": [ 27 | str(v) 28 | for v in pyproject["tool"]["poetry"]["group"]["dev"][ 29 | "dependencies" 30 | ].values() 31 | ], 32 | }, 33 | } 34 | 35 | 36 | setup(**build_setup_args()) 37 | -------------------------------------------------------------------------------- /src/cnc/__init__.py: -------------------------------------------------------------------------------- 1 | from .models import ( 2 | AppConfig, 3 | Application, 4 | EnvironmentVariable, 5 | Environment, 6 | EnvironmentCollection, 7 | BuildStageManager, 8 | DeployStageManager, 9 | ProvisionStageManager, 10 | ) 11 | -------------------------------------------------------------------------------- /src/cnc/check_dependencies.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import sys 3 | 4 | from .logger import get_logger 5 | 6 | log = get_logger(__name__) 7 | 8 | 9 | def check_deps(application=None): 10 | # add provider-specific ones 11 | # check bash shell available 12 | # print info of what version of each is installed 13 | # print tested versions? 14 | 15 | dep_list = ["terraform", "jq"] 16 | 17 | if application.provider == "gcp": 18 | dep_list.append("gcloud") 19 | elif application.provider == "aws": 20 | dep_list.append("aws") 21 | 22 | for tool_name in dep_list: 23 | check_if_installed(tool_name) 24 | 25 | 26 | def check_if_installed(tool_name): 27 | if shutil.which(tool_name) is None: 28 | log.warning(f"Error: The required program '{tool_name}' is not installed.") 29 | sys.exit(1) 30 | -------------------------------------------------------------------------------- /src/cnc/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/src/cnc/commands/__init__.py -------------------------------------------------------------------------------- /src/cnc/commands/build.py: -------------------------------------------------------------------------------- 1 | import typer 2 | import time 3 | from typing import List 4 | from typing_extensions import Annotated 5 | 6 | from cnc.models import BuildStageManager 7 | from .telemetry import send_event 8 | 9 | from cnc.logger import get_logger 10 | 11 | log = get_logger(__name__) 12 | 13 | 14 | app = typer.Typer() 15 | 16 | 17 | @app.command() 18 | def perform( 19 | ctx: typer.Context, 20 | environment_name: str, 21 | service_tags: List[str] = typer.Option( 22 | [], 23 | "--service-tag", 24 | "-t", 25 | help="Set the tag to use for this service with svc_name=tag, default is 'int(time.time())'. If any provided, only builds provided services and will build all services if empty", 26 | ), 27 | default_tag: Annotated[ 28 | str, 29 | typer.Option( 30 | "-d", "--default-tag", envvar="CNC_DEFAULT_TAG", help="CNC default tag" 31 | ), 32 | ] = None, 33 | collection_name: str = "", 34 | cleanup: bool = True, 35 | debug: bool = False, 36 | generate: bool = True, 37 | webhook_url: str = typer.Option( 38 | None, 39 | "--webhook-url", 40 | help="Webhook URL for sending build notifications", 41 | ), 42 | webhook_token: str = typer.Option( 43 | None, 44 | "--webhook-token", 45 | help="Webhook token for authentication", 46 | ), 47 | parallel: bool = typer.Option( 48 | False, 49 | "--parallel", 50 | "-p", 51 | help="Enable parallel build", 52 | ), 53 | ): 54 | """Build containers for config-defined services""" 55 | start_time = time.time() 56 | send_event("build.perform") 57 | collection = ctx.obj.application.collection_by_name(collection_name) 58 | if not collection: 59 | log.error(f"No collection found for: {collection_name}") 60 | raise typer.Exit(code=1) 61 | 62 | environment = collection.environment_by_name(environment_name) 63 | if not environment: 64 | log.error(f"No environment found for: {environment_name}") 65 | raise typer.Exit(code=1) 66 | 67 | builder = BuildStageManager( 68 | environment, 69 | service_tags=service_tags, 70 | default_tag=default_tag, 71 | webhook_url=webhook_url, 72 | webhook_token=webhook_token, 73 | parallel_exec_enabled=parallel, 74 | ) 75 | cmd_exit_code = builder.perform( 76 | should_cleanup=cleanup, 77 | should_regenerate_config=generate, 78 | debug=debug, 79 | ) 80 | 81 | log.debug( 82 | f"All set building for {builder.config_files_path} in " 83 | f"{int(start_time - time.time())} seconds" 84 | ) 85 | raise typer.Exit(code=cmd_exit_code) 86 | -------------------------------------------------------------------------------- /src/cnc/commands/deploy.py: -------------------------------------------------------------------------------- 1 | import typer 2 | import time 3 | from typing import List 4 | from typing_extensions import Annotated 5 | 6 | from cnc.models import DeployStageManager 7 | from .telemetry import send_event 8 | 9 | from cnc.logger import get_logger 10 | 11 | log = get_logger(__name__) 12 | 13 | app = typer.Typer() 14 | 15 | 16 | @app.command() 17 | def perform( 18 | ctx: typer.Context, 19 | environment_name: str, 20 | service_tags: List[str] = typer.Option( 21 | [], 22 | "--service-tag", 23 | "-t", 24 | help="Set the tag to use for this service with svc_name=tag, default is 'int(time.time())'. If any provided, only builds provided services and will build all services if empty", 25 | ), 26 | default_tag: Annotated[ 27 | str, 28 | typer.Option( 29 | "-d", "--default-tag", envvar="CNC_DEFAULT_TAG", help="CNC default tag" 30 | ), 31 | ] = None, 32 | collection_name: str = "", 33 | cleanup: bool = True, 34 | debug: bool = False, 35 | generate: bool = True, 36 | webhook_url: str = typer.Option( 37 | None, 38 | "--webhook-url", 39 | help="Webhook URL for sending build notifications", 40 | ), 41 | webhook_token: str = typer.Option( 42 | None, 43 | "--webhook-token", 44 | help="Webhook token for authentication", 45 | ), 46 | ): 47 | start_time = time.time() 48 | send_event("deploy.perform") 49 | collection = ctx.obj.application.collection_by_name(collection_name) 50 | if not collection: 51 | log.error(f"No collection found for: {collection_name}") 52 | raise typer.Exit(code=1) 53 | 54 | environment = collection.environment_by_name(environment_name) 55 | if not environment: 56 | log.error(f"No environment found for: {environment_name}") 57 | raise typer.Exit(code=1) 58 | 59 | deployer = DeployStageManager( 60 | environment, 61 | service_tags=service_tags, 62 | default_tag=default_tag, 63 | webhook_url=webhook_url, 64 | webhook_token=webhook_token, 65 | ) 66 | cmd_exit_code = deployer.perform( 67 | should_cleanup=cleanup, 68 | should_regenerate_config=generate, 69 | debug=debug, 70 | ) 71 | 72 | log.debug( 73 | f"All set deploying for {deployer.rendered_files_path} in " 74 | f"{int(time.time() - start_time)} seconds" 75 | ) 76 | raise typer.Exit(code=cmd_exit_code) 77 | -------------------------------------------------------------------------------- /src/cnc/commands/render.py: -------------------------------------------------------------------------------- 1 | import typer 2 | from cnc.models import ProvisionStageManager, DeployStageManager, BuildStageManager 3 | from .telemetry import send_event 4 | 5 | from cnc.logger import get_logger 6 | 7 | log = get_logger(__name__) 8 | 9 | app = typer.Typer() 10 | 11 | 12 | @app.callback(invoke_without_command=True) 13 | @app.command() 14 | def render( 15 | ctx: typer.Context, 16 | stage: str = "provision", 17 | extra_context: dict = None, 18 | ): 19 | send_event("render.render") 20 | 21 | manager_map = { 22 | "build": BuildStageManager, 23 | "provision": ProvisionStageManager, 24 | "deploy": DeployStageManager, 25 | } 26 | 27 | if not manager_map.get("stage"): 28 | log.error(f"No manager for {stage}") 29 | raise typer.Exit(code=1) 30 | 31 | # get manager 32 | # render template with added context 33 | 34 | raise typer.Exit() 35 | -------------------------------------------------------------------------------- /src/cnc/commands/shell.py: -------------------------------------------------------------------------------- 1 | import typer 2 | from IPython import embed 3 | from cnc.models import ( 4 | # Environment, 5 | # EnvironmentVariable, 6 | # AppConfig, 7 | # EnvironmentCollection, 8 | Application, 9 | ) 10 | from .telemetry import send_event 11 | 12 | from cnc.logger import get_logger 13 | 14 | log = get_logger(__name__) 15 | app = typer.Typer() 16 | 17 | 18 | def get_app(): 19 | app = Application.from_environments_yml("environments.yml") 20 | return app 21 | 22 | 23 | @app.callback(invoke_without_command=True) 24 | @app.command() 25 | def start(ctx: typer.Context): 26 | send_event("shell.start") 27 | local_vars = { 28 | "get_app": get_app, 29 | "parse_yml": get_app, 30 | } 31 | embed(user_ns=local_vars, colors="neutral") 32 | raise typer.Exit() 33 | -------------------------------------------------------------------------------- /src/cnc/commands/telemetry.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import rudderstack.analytics as rudder_analytics 4 | import machineid 5 | 6 | rudder_analytics.write_key = "2eEyoetBjPLeYUOg8jRfOicBRiS" 7 | rudder_analytics.dataPlaneUrl = "https://withcoherepvm.dataplane.rudderstack.com" 8 | rudder_analytics.debug = True 9 | rudder_analytics.gzip = True 10 | 11 | # default to enabling telemetry for now 12 | TELEMETRY_ENABLED = ( 13 | False 14 | if os.environ.get("CNC_TELEMETRY_DISABLED") 15 | in [1, "1", "true", "True", "TRUE", "yes", "Y", "y", "Yes", "YES"] 16 | else True 17 | ) 18 | CURRENT_COMMAND = "" 19 | 20 | 21 | def send_event(command_name: str): 22 | command_data = { 23 | "name": command_name, 24 | } 25 | if TELEMETRY_ENABLED: 26 | try: 27 | _id = machineid.id() 28 | except Exception as e: 29 | print(f"Cannot get machine ID: {e}") 30 | _id = None 31 | 32 | try: 33 | user_id = os.environ.get("CNC_USER_ID", _id or "UNKNOWN") 34 | print(f"Sending {command_data} to RS for {user_id}") 35 | rudder_analytics.track(user_id, "command", command_data) 36 | except Exception as e: 37 | print(f"Cannot send telemetry event: {e}") 38 | -------------------------------------------------------------------------------- /src/cnc/commands/template_editor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/src/cnc/commands/template_editor/__init__.py -------------------------------------------------------------------------------- /src/cnc/commands/update.py: -------------------------------------------------------------------------------- 1 | import typer 2 | import time 3 | from typing import List 4 | from typing_extensions import Annotated 5 | 6 | from cnc.models import DeployStageManager 7 | from cnc.models import BuildStageManager 8 | from .telemetry import send_event 9 | 10 | from cnc.logger import get_logger 11 | 12 | log = get_logger(__name__) 13 | 14 | 15 | app = typer.Typer() 16 | 17 | 18 | @app.command() 19 | def perform( 20 | ctx: typer.Context, 21 | environment_name: str, 22 | service_tags: List[str] = typer.Option( 23 | [], 24 | "--service-tag", 25 | "-t", 26 | help="Set the tag to use for this service with svc_name=tag, default is 'int(time.time())'. If any provided, only builds provided services and will build all services if empty", 27 | ), 28 | default_tag: Annotated[ 29 | str, 30 | typer.Option( 31 | "-d", "--default-tag", envvar="CNC_DEFAULT_TAG", help="CNC default tag" 32 | ), 33 | ] = None, 34 | collection_name: str = "", 35 | cleanup: bool = True, 36 | debug: bool = False, 37 | generate: bool = True, 38 | ): 39 | start_time = time.time() 40 | send_event("update.perform") 41 | collection = ctx.obj.application.collection_by_name(collection_name) 42 | if not collection: 43 | log.error(f"No collection found for: {collection_name}") 44 | raise typer.Exit(code=1) 45 | 46 | environment = collection.environment_by_name(environment_name) 47 | if not environment: 48 | log.error(f"No environment found for: {environment_name}") 49 | raise typer.Exit(code=1) 50 | 51 | builder = BuildStageManager( 52 | environment, 53 | service_tags=service_tags, 54 | default_tag=default_tag, 55 | ) 56 | _ret = builder.perform( 57 | should_cleanup=cleanup, 58 | should_regenerate_config=generate, 59 | debug=debug, 60 | ) 61 | 62 | if _ret == 0: 63 | deployer = DeployStageManager( 64 | environment, 65 | service_tags=service_tags, 66 | default_tag=default_tag, 67 | ) 68 | deployer.perform( 69 | should_cleanup=cleanup, 70 | should_regenerate_config=generate, 71 | debug=debug, 72 | ) 73 | else: 74 | log.warning(f"Build failed (exit code {_ret}), did not deploy!") 75 | 76 | log.debug( 77 | f"All set updating for {service_tags}/{default_tag} in {int(time.time() - start_time)} seconds" 78 | ) 79 | raise typer.Exit() 80 | -------------------------------------------------------------------------------- /src/cnc/constants.py: -------------------------------------------------------------------------------- 1 | from typing import ClassVar 2 | 3 | 4 | class EnvironmentVariableTypes: 5 | VARIABLE_TYPE_SECRET: ClassVar[str] = "secret" 6 | VARIABLE_TYPE_OUTPUT: ClassVar[str] = "output" 7 | VARIABLE_TYPE_STANDARD: ClassVar[str] = "standard" 8 | VARIABLE_TYPE_ALIAS: ClassVar[str] = "alias" 9 | 10 | @classmethod 11 | def allowed_types(cls): 12 | return [ 13 | cls.VARIABLE_TYPE_STANDARD, 14 | cls.VARIABLE_TYPE_SECRET, 15 | cls.VARIABLE_TYPE_OUTPUT, 16 | cls.VARIABLE_TYPE_ALIAS, 17 | ] 18 | 19 | 20 | class EnvironmentVariableDestinations: 21 | VARIABLE_TYPE_NAME: ClassVar[str] = "name" 22 | VARIABLE_TYPE_FILE: ClassVar[str] = "file" 23 | 24 | @classmethod 25 | def allowed_types(cls): 26 | return [cls.VARIABLE_TYPE_NAME, cls.VARIABLE_TYPE_FILE] 27 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/build/build_lambda_functions.sh.j2: -------------------------------------------------------------------------------- 1 | build_{{ service.name }} () { 2 | echo "Verify Lambda function code..." 3 | 4 | source_dir="{{ service.build.context }}" 5 | 6 | if [ -d "$source_dir" ]; then 7 | echo "Source is a directory: $source_dir" 8 | elif [ -f "$source_dir" ]; then 9 | echo "Source is a file: $source_dir" 10 | else 11 | echo "Error: Source path is not valid: $source_dir" 12 | exit 1 13 | fi 14 | } 15 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/build/main.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | trap 'send_{{ service.name }}_status_hook $?' EXIT 5 | 6 | if ! command -v bash >/dev/null 2>&1; then 7 | echo "Bash not found. Exiting." 8 | exit 1 9 | fi 10 | 11 | {% block install_commands %} 12 | source {{ builder.rendered_files_path }}/build-{{ service.name }}-functions.sh 13 | send_{{ service.name }}_status_hook 0 "working" 14 | {% endblock %} 15 | 16 | {% if not service.is_serverless %} 17 | 18 | {% block pre_build_commands %} 19 | {% if service.build %} 20 | login_to_registries 21 | {% endif %} 22 | {% endblock %} 23 | 24 | {% block build_commands %} 25 | log_{{ service.name }}_variables 26 | 27 | {%- if service.image %} 28 | verify_{{ service.name }}_image_exists 29 | {%- else %} 30 | build_{{ service.name }}_image 31 | 32 | {%- if service.build and not builder.file_exists(service.build.dockerfile) %} 33 | push_{{ service.name }}_image_tags 34 | {%- endif %} 35 | 36 | {%- endif %} 37 | 38 | {% endblock %} 39 | 40 | {% block post_build_commands %} 41 | {% endblock %} 42 | 43 | {% endif %} 44 | 45 | {% if service.is_serverless %} 46 | {% block serverless_block %} 47 | build_{{ service.name }} 48 | {% endblock %} 49 | {% endif %} -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/base.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | trap 'send_{{ service.name }}_status_hook $?' EXIT 5 | 6 | {% block install_commands %} 7 | source {{ deployer.rendered_files_path }}/deploy-{{ service.name }}-functions.sh 8 | send_{{ service.name }}_status_hook 0 "working" 9 | {% endblock %} 10 | 11 | {% block build_commands %} 12 | 13 | {% if service.is_backend %} 14 | 15 | {% if service.settings.migrate %} 16 | run_{{ service.name }}_migrate 17 | {% endif %} 18 | 19 | {% if (not service.environment.is_production) and load_database_snapshot %} 20 | load_snapshot 21 | {% endif %} 22 | 23 | {% if not service.environment.is_production and service.settings.seed %} 24 | seed_databases 25 | {% endif %} 26 | 27 | deploy_{{ service.name }}_to_ecs 28 | 29 | {% if service.environment.is_production %} 30 | 31 | check_{{ service.name }}_ecs_service_deploy_status 32 | 33 | {% if service.settings.workers %} 34 | deploy_{{ service.name }}_workers_to_ecs 35 | check_{{ service.name }}_ecs_workers_deploy_status 36 | {% endif %} 37 | 38 | {% else %} 39 | 40 | {% if service.settings.workers %} 41 | deploy_{{ service.name }}_workers_to_ecs 42 | {% endif %} 43 | 44 | check_{{ service.name }}_ecs_service_deploy_status 45 | {% if service.settings.workers %} 46 | check_{{ service.name }}_ecs_workers_deploy_status 47 | {% endif %} 48 | 49 | {% endif %} 50 | 51 | {% if service.settings.scheduled_tasks %} 52 | deploy_{{ service.name }}_scheduled_tasks 53 | {% endif %} 54 | 55 | {% elif service.is_frontend %} 56 | 57 | build_{{ service.name }}_frontend_assets 58 | deploy_{{ service.name }}_to_s3 59 | 60 | {% endif %} 61 | 62 | {% endblock %} 63 | 64 | {% block finally_build_commands %} 65 | {% endblock %} 66 | 67 | {% if service.is_serverless %} 68 | {% block serverless_block %} 69 | {% include "deploy_lambda_functions.sh.j2" %} 70 | deploy_{{ service.name }} 71 | {% endblock %} 72 | {% endif %} 73 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/deploy_functions.sh.j2: -------------------------------------------------------------------------------- 1 | {% extends "deploy_functions_base.sh.j2" %} -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/ecs_web_task.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "family": "{{ task_name }}", 3 | "cpu": "{{ service.deploy.resources.limits.cpu }}", 4 | "memory": "{{ service.deploy.resources.limits.memory }}", 5 | "requiresCompatibilities": ["FARGATE"], 6 | "networkMode": "awsvpc", 7 | "taskRoleArn": "{{ environment.collection.task_execution_role_arn }}", 8 | "executionRoleArn": "{{ environment.collection.task_execution_role_arn }}", 9 | "containerDefinitions": [ 10 | { 11 | "name": "{{ service.instance_name }}", 12 | "image": "{{ service.image_for_tag(deployer.tag_for_service(service.name) or 'latest') }}", 13 | "ulimits": [ 14 | { 15 | "name": "nofile", 16 | "hardLimit": 65535, 17 | "softLimit": 65535 18 | } 19 | ], 20 | {% if command %} 21 | "entryPoint": ["sh"], 22 | "command": [ 23 | "-c", 24 | "{{ command }}" 25 | ], 26 | {% endif %} 27 | {% if expose_ports %} 28 | "portMappings": [ 29 | { 30 | "containerPort": 8080, 31 | "hostPort": 8080 32 | } 33 | ], 34 | {% endif %} 35 | "logConfiguration": { 36 | "logDriver": "awslogs", 37 | "options": { 38 | "awslogs-group": "{{ service.instance_name }}", 39 | "awslogs-region": "{{ environment.collection.region }}", 40 | "awslogs-stream-prefix": "{{ service.log_stream_prefix('web') }}" 41 | } 42 | }, 43 | "secrets": [ 44 | {% for item in service.environment_secrets %} 45 | { 46 | "name": "{{ item.name }}", 47 | "valueFrom": "arn:aws:secretsmanager:{{ environment.collection.region }}:{{ environment.collection.account_id }}:secret:{{ item.secret_id }}" 48 | }{% if not loop.last %},{% endif %} 49 | {% endfor %} 50 | ], 51 | "environment": [ 52 | { 53 | "name": "PORT", 54 | "value": "8080" 55 | }{% if service.environment_variables %},{% endif %} 56 | {% for item in service.insecure_environment_items %} 57 | { 58 | "name": "{{ item.name }}", 59 | "value": {{ item.value | tojson }} 60 | }{% if not loop.last %},{% endif %} 61 | {% endfor %} 62 | ] 63 | } 64 | ] 65 | } -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/env_vars.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "Variables": { 3 | {%- for item in service.insecure_environment_items %} 4 | "{{ item.name }}": {{ item.value | tojson }}{%- if not loop.last or service.environment_secrets %},{%- endif %} 5 | {%- endfor %} 6 | 7 | {%- for secret in service.environment_secrets %} 8 | {% if service.settings.secrets_mode == 'plaintext' %} 9 | "{{ secret.name }}": {{ secret.value | tojson }}{%- if not loop.last %},{%- endif %} 10 | {%- elif service.settings.secrets_mode == 'arn' %} 11 | "{{ secret.name }}": "{{ secret.secret_id }}"{%- if not loop.last %},{%- endif %} 12 | {%- endif %} 13 | {%- endfor %} 14 | } 15 | } -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/main.sh.j2: -------------------------------------------------------------------------------- 1 | {% extends "base.sh.j2" %} 2 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/partials/backend_deploy.sh.j2: -------------------------------------------------------------------------------- 1 | {# Backend deploy logic - only kicks off deploy, does not report success/fail 2 | see backend_deploy_status_check.sh.j2 #} 3 | 4 | echo -e \\nDeploying {{ task_name }} to Amazon ECS...\\n 5 | 6 | task_revision=$(cat {{ deployer.rendered_files_path }}/task_rev_{{ task_name }}.txt) 7 | task_rev={{ task_name }}:$task_revision 8 | 9 | init_describe_result=$(aws ecs describe-services --region "{{ environment.collection.region }}" --cluster {{ environment.collection.instance_name }} --services {{ task_name }}) 10 | current_desired_count=$(echo $init_describe_result | jq '.services[0].desiredCount') 11 | min_desired_count={{ min_scale }} 12 | if [ $current_desired_count -lt $min_desired_count ]; then DESIRED_COUNT=$min_desired_count; else DESIRED_COUNT=$current_desired_count; fi 13 | update_result=$(aws ecs update-service --region "{{ environment.collection.region }}" --cluster {{ environment.collection.instance_name }} --service {{ task_name }} --task-definition $task_rev --desired-count $DESIRED_COUNT --force-new-deployment) 14 | updated_task_revision=$(echo $update_result | jq '.service.deployments[0].taskDefinition') 15 | updated_task_rev=$(echo $updated_task_revision | sed -e 's/^"//' -e 's/"$//' -e 's/^.\+\///') 16 | 17 | if [ "${COHERENCE_BUILD_DEBUG_MODE:-0}" -eq 1 ]; then 18 | echo -e \\nUpdateService response:\\n 19 | echo $update_result 20 | echo $updated_task_revision 21 | echo $updated_task_rev 22 | fi 23 | 24 | if [ "$updated_task_rev" != "$task_rev" ]; then 25 | echo -e "\\nDeploy failed... check events and stopped tasks in the {{ task_name }} ecs service for more information.\\n" 26 | exit 1 27 | fi 28 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/partials/backend_deploy_status_check.sh.j2: -------------------------------------------------------------------------------- 1 | {# Backend deploy status check logic - only checks status of deploy, does not kick off deploy 2 | see backend_deploy.yml.jinja2 #} 3 | echo -e "\\nChecking deployment status for {{ task_name }}...\\n" 4 | task_revision=$(cat {{ deployer.rendered_files_path }}/task_rev_{{ task_name }}.txt) 5 | task_rev={{ task_name }}:$task_revision 6 | {% with 7 | wait_for="services-stable", 8 | wait_task_name="Deployment", 9 | wait_cmd_args=("--services " + task_name), 10 | num_retries=(service.settings.timeouts.deploy//10) 11 | %} 12 | {% include "partials/ecs_wait_with_retries.sh.j2" %} 13 | {% endwith %} 14 | 15 | describe_result=$(aws ecs describe-services --region "{{ environment.collection.region }}" --cluster {{ environment.collection.instance_name }} --services {{ task_name }}) 16 | primary_deploy_status=$(echo $describe_result | jq '.services[0].deployments[] | select(.status=="PRIMARY") | .rolloutState') 17 | primary_deploy_status=$(echo $primary_deploy_status | sed 's/"//g') 18 | 19 | max_retries=9 20 | count=0 21 | 22 | while [ "$primary_deploy_status" == "IN_PROGRESS" ] 23 | do 24 | count=`expr $count + 1` 25 | if [ $count -eq $max_retries ] 26 | then 27 | exit 1 28 | fi 29 | # Its possible to be considered "stable" while still in progress.. 30 | # sleep to allow time for healthchecks to pass 31 | echo -e \\nDeployment still in progress, waiting for healthchecks to pass...\\n 32 | sleep 10 33 | describe_result=$(aws ecs describe-services --region "{{ environment.collection.region }}" --cluster {{ environment.collection.instance_name }} --services {{ task_name }}) 34 | primary_deploy_status=$(echo $describe_result | jq '.services[0].deployments[] | select(.status=="PRIMARY") | .rolloutState') 35 | primary_deploy_status=$(echo $primary_deploy_status | sed 's/"//g') 36 | done 37 | 38 | final_task_revision=$(echo $describe_result | jq '.services[0].deployments[] | select(.status=="PRIMARY") | .taskDefinition') 39 | final_task_rev=$(echo $final_task_revision | sed -e 's/^"//' -e 's/"$//' -e 's/^.\+\///') 40 | 41 | if [ "${COHERENCE_BUILD_DEBUG_MODE:-0}" -eq 1 ]; then 42 | echo $describe_result 43 | echo $final_task_rev 44 | echo $primary_deploy_status 45 | fi 46 | 47 | if [ "$final_task_rev" != "$task_rev" ] || [ "$primary_deploy_status" != "COMPLETED" ]; then 48 | echo -e "\\nDeploy failed... check events and stopped tasks in the {{ task_name }} ecs service for more information.\\n" 49 | exit 1 50 | fi 51 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/partials/ecs_wait_with_retries.sh.j2: -------------------------------------------------------------------------------- 1 | {# Retry logic for ecs wait commands (the aws cli command has a timeout of 10 mins) #} 2 | max_retries={{ num_retries or 2 }} 3 | count=0 4 | wait_command="aws ecs wait {{ wait_for }} --region {{ environment.collection.region }} --cluster {{ environment.collection.instance_name }} {{ wait_cmd_args }}" 5 | 6 | until $wait_command 7 | do 8 | wait_result=$? 9 | count=`expr $count + 1` 10 | if [ $wait_result -ne 255 ] 11 | then 12 | echo -e "\\n{{ wait_task_name }} failed, something went wrong while waiting for status.\\n" 13 | exit 1 14 | elif [ $count -eq $max_retries ] 15 | then 16 | echo -e "\\n{{ wait_task_name }} timed out\\n" 17 | exit 1 18 | fi 19 | echo -e "\\naws ecs wait command timed out, retrying...\\n" 20 | done 21 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/partials/update_task_definition.sh.j2: -------------------------------------------------------------------------------- 1 | echo "Updating task definition for {{ task_name }}..." 2 | task_update_result=$(aws ecs register-task-definition --cli-input-json file://{{ deployer.rendered_files_path }}/{{ task_filepath }}) 3 | task_update_resp_code=$? 4 | 5 | if [ $task_update_resp_code -ne 0 ]; then 6 | echo $task_update_result 7 | exit $task_update_resp_code 8 | fi 9 | 10 | TASK_REV_{{ task_name.replace("-", "_") }}=$(echo $task_update_result | jq '.taskDefinition.revision') 11 | echo $TASK_REV_{{ task_name.replace("-", "_") }} > {{ deployer.rendered_files_path }}/task_rev_{{ task_name }}.txt 12 | echo "Task definition for {{ task_name }} updated successfully." 13 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/pre_deploy.sh.j2: -------------------------------------------------------------------------------- 1 | source {{ deployer.rendered_files_path }}/pre-deploy-{{ environment.name }}-functions.sh 2 | 3 | {% for service in deployer.services %} 4 | {% if service.is_backend %} 5 | update_{{ service.name }}_ecs_task_definitions 6 | {% endif %} 7 | {% endfor %} 8 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/deploy/pre_deploy_functions.sh.j2: -------------------------------------------------------------------------------- 1 | {% for service in deployer.services %} 2 | {% if service.is_backend %} 3 | update_{{ service.name }}_ecs_task_definitions () { 4 | {% with 5 | task_name=service.instance_name, 6 | task_filepath="ecs-web-{}.json".format(service.name) 7 | %} 8 | {% include "partials/update_task_definition.sh.j2" %} 9 | {% endwith %} 10 | 11 | {% if service.settings.workers %} 12 | {% for worker in service.settings.workers %} 13 | {% with 14 | task_name='_'.join([service.instance_name, worker.name]), 15 | task_filepath="ecs-worker-{}-{}.json".format(service.name, worker.name) 16 | %} 17 | {% include "partials/update_task_definition.sh.j2" %} 18 | {% endwith %} 19 | {% endfor %} 20 | {% endif %} 21 | 22 | {% if service.settings.scheduled_tasks %} 23 | {% for task in service.settings.scheduled_tasks %} 24 | {% with 25 | task_name='_'.join([service.instance_name, task.name]), 26 | task_filepath="ecs-sched-{}-{}.json".format(service.name, task.name) 27 | %} 28 | {% include "partials/update_task_definition.sh.j2" %} 29 | {% endwith %} 30 | {% endfor %} 31 | {% endif %} 32 | } 33 | {% endif %} 34 | {% endfor %} 35 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/base.tf.j2: -------------------------------------------------------------------------------- 1 | {# Initialize lb_rules.priority globally once #} 2 | {% set lb_rules = namespace(priority=1) %} 3 | 4 | {% block state %} 5 | terraform { 6 | backend "local" { 7 | path = "{{ config_renderer.working_dir }}/.terraform/{{ config_renderer.tf_state_namespace }}.tfstate" 8 | } 9 | } 10 | {% endblock %} 11 | 12 | {% if not output_only %} 13 | 14 | {% block providers %} 15 | {% include "partials/collection/providers.tf.j2" %} 16 | {% endblock providers %} 17 | 18 | {% block collection %}{% endblock collection %} 19 | {% block collection_level_resources %} 20 | {% include "partials/collection_level_resources.tf.j2" %} 21 | 22 | {% if env_collection.hosted_zone_ns_records %} 23 | {% if env_collection.serverless_services %} 24 | {% block lambda_shared %} 25 | {% include "partials/collection/lambda/lambda.tf.j2" %} 26 | {% include "partials/collection/lambda/variables.tf.j2" %} 27 | {% include "partials/collection/lambda/outputs.tf.j2" %} 28 | {% endblock lambda_shared %} 29 | {% endif %} 30 | 31 | {% if env_collection.dynamodb_resources %} 32 | {% block dynamodb_shared %} 33 | {% include "partials/collection/dynamodb/dynamodb.tf.j2" %} 34 | {% include "partials/collection/dynamodb/variables.tf.j2" %} 35 | {% include "partials/collection/dynamodb/outputs.tf.j2" %} 36 | {% endblock dynamodb_shared %} 37 | {% endif %} 38 | 39 | {% if env_collection.database_resources %} 40 | {% block database_shared %} 41 | {% include "partials/collection/database/rds.tf.j2" %} 42 | {% endblock database_shared %} 43 | {% endif %} 44 | {% endif %} 45 | {% endblock collection_level_resources %} 46 | 47 | {% for service in sorted(env_collection.all_web_services, key=svc_path_lambda, reverse=True) %} 48 | {% set environment = service.environment %} 49 | 50 | {% block service_resources scoped %} 51 | {% include "partials/service_level_resources.tf.j2" %} 52 | {% endblock service_resources %} 53 | {% endfor %} 54 | 55 | 56 | {# For active environment #} 57 | {% for environment in env_collection.active_environments %} 58 | {% block environment_resources scoped %} 59 | {% include "partials/environment_level_resources.tf.j2" %} 60 | {% endblock environment_resources %} 61 | {% endfor %} 62 | 63 | {% set efs_mountpoint_status = namespace(created=False) %} 64 | 65 | 66 | {% block custom scoped %} 67 | {% endblock custom %} 68 | {% endif %}{# {% if not output_only %} #} 69 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/frontend_routing_lambda/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | exports.handler = async (event, context, callback) => { 4 | const cf = event.Records[0].cf; 5 | const request = cf.request; 6 | const response = cf.response; 7 | const statusCode = response.status; 8 | 9 | if (statusCode == '404') { 10 | response.status = '200' 11 | } 12 | 13 | console.log('response: ' + JSON.stringify(response)); 14 | callback(null, response); 15 | return response; 16 | }; 17 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/main.tf.j2: -------------------------------------------------------------------------------- 1 | {% extends "base.tf.j2" %} 2 | 3 | {# All Available Blocks #} 4 | 5 | {# Env collection level resource #} 6 | {# {% block application %} #} 7 | {# {% endblock application %} #} 8 | 9 | {# Per active environment #} 10 | {# {% block environment_resources %} #} 11 | {# {% endblock environment_resources %} #} 12 | 13 | {# Per active service #} 14 | {# {% block service_resources %} #} 15 | {# {% endblock service_resources %} #} 16 | 17 | {# Per active|paused environment #} 18 | {# {% block active_and_paused_envs %} #} 19 | {# {% endblock active_and_paused_envs %} #} 20 | 21 | {# Custom tf #} 22 | {# {% block custom %} #} 23 | {# {% endblock custom %} #} 24 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/partials/collection/has_active_frontend_deployments.tf.j2: -------------------------------------------------------------------------------- 1 | data "aws_iam_policy_document" "lambda_exec_role" { 2 | statement { 3 | effect = "Allow" 4 | actions = ["sts:AssumeRole"] 5 | principals { 6 | type = "Service" 7 | identifiers = ["lambda.amazonaws.com", "edgelambda.amazonaws.com"] 8 | } 9 | } 10 | } 11 | 12 | resource "aws_iam_role" "{{ env_collection.instance_name }}_lambda" { 13 | name = "{{ env_collection.instance_name }}-lambda" 14 | assume_role_policy = "${data.aws_iam_policy_document.lambda_exec_role.json}" 15 | } 16 | 17 | resource "aws_lambda_function" "{{ env_collection.instance_name }}_frontend_routing" { 18 | function_name = "{{ env_collection.instance_name }}-frontend-routing" 19 | publish = true 20 | provider = aws.us_east_1 21 | role = aws_iam_role.{{ env_collection.instance_name }}_lambda.arn 22 | handler = "index.handler" 23 | source_code_hash = filebase64sha256("frontend_routing_lambda.zip") 24 | filename = "frontend_routing_lambda.zip" 25 | runtime = "nodejs16.x" 26 | } 27 | 28 | resource "aws_cloudfront_origin_request_policy" "{{ env_collection.instance_name }}_frontend" { 29 | name = "{{ env_collection.instance_name }}-frontend" 30 | comment = "Frontend origin request policy" 31 | cookies_config { 32 | cookie_behavior = "all" 33 | } 34 | headers_config { 35 | header_behavior = "none" 36 | } 37 | query_strings_config { 38 | query_string_behavior = "all" 39 | } 40 | } 41 | 42 | resource "aws_cloudfront_cache_policy" "{{ env_collection.instance_name }}_frontend" { 43 | name = "{{ env_collection.instance_name }}-frontend" 44 | comment = "Frontend cache policy" 45 | default_ttl = 0 46 | parameters_in_cache_key_and_forwarded_to_origin { 47 | cookies_config { 48 | cookie_behavior = "all" 49 | } 50 | headers_config { 51 | header_behavior = "none" 52 | } 53 | query_strings_config { 54 | query_string_behavior = "all" 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/partials/collection_level_resources.tf.j2: -------------------------------------------------------------------------------- 1 | resource "aws_route53_zone" "{{ env_collection.instance_name }}" { 2 | name = "{{ env_collection.base_domain }}" 3 | } 4 | 5 | output "hosted_zone_ns_records" { 6 | value = "${aws_route53_zone.{{ env_collection.instance_name }}.name_servers}" 7 | } 8 | 9 | {% if env_collection.hosted_zone_ns_records %} 10 | 11 | {% if env_collection.load_database_snapshot %} 12 | data "aws_iam_policy_document" "{{ env_collection.instance_name }}_snapshot_loader" { 13 | statement { 14 | effect = "Allow" 15 | resources = [{% for path in env_collection.snapshot_file_paths %}"arn:aws:s3:::{{ path }}"{% if not loop.last %}, {% endif %}{% endfor %}] 16 | actions = ["s3:*"] 17 | } 18 | } 19 | 20 | resource "aws_iam_role_policy" "{{ env_collection.instance_name }}_snapshot_loader" { 21 | name = "{{ app.name }}_snapshot_ecs_execution_role_policy" 22 | policy = "${data.aws_iam_policy_document.{{ env_collection.instance_name }}_snapshot_loader.json}" 23 | role = "${aws_iam_role.{{ env_collection.instance_name }}_ecs.id}" 24 | } 25 | {% endif %} {# if env_collection.load_database_snapshot #} 26 | 27 | {# {% if env_collection.frontend_services %}{% endif %}if env_collection.frontend_services #} 28 | 29 | {% if env_collection.backend_services %} 30 | {% include "partials/collection/vpc_configuration.tf.j2" %} 31 | {% include "partials/collection/has_backend_services.tf.j2" %} 32 | {% endif %}{# if env_collection.backend_services #} 33 | 34 | {% if env_collection.has_active_deployments %} 35 | {% include "partials/collection/has_active_deployments.tf.j2" %} 36 | {% if env_collection.frontend_services %} 37 | {% include "partials/collection/has_active_frontend_deployments.tf.j2" %} 38 | {% endif %}{# if env_collection.frontend_services #} 39 | {% if env_collection.backend_services %} 40 | {% include "partials/collection/has_active_backend_deployments.tf.j2" %} 41 | {% endif %}{# if env_collection.backend_services #} 42 | {% endif %} {# if env_collection.has_active_deployments #} 43 | 44 | {% endif %} {# if hosted_zone_ns_records #} 45 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/partials/environment/efs.tf.j2: -------------------------------------------------------------------------------- 1 | {% for resource in environment.filesystem_resources %} 2 | {% if not efs_mountpoint_status.created %} 3 | {% if resource.settings.existing_filesystem_id %} 4 | resource "aws_efs_mount_target" "{{ resource.settings.existing_filesystem_id }}" { 5 | count = length(concat( 6 | local.private_subnet_ids, 7 | length(local.private_subnet_ids) > 0 ? [] : aws_subnet.{{ env_collection.instance_name }}_private.*.id 8 | )) 9 | subnet_id = element(concat( 10 | local.private_subnet_ids, 11 | length(local.private_subnet_ids) > 0 ? [] : aws_subnet.{{ env_collection.instance_name }}_private.*.id 12 | ), count.index) 13 | file_system_id = "{{ resource.settings.existing_filesystem_id }}" 14 | security_groups = [aws_security_group.{{ env_collection.instance_name }}-efs-access.id] 15 | depends_on = [aws_security_group.{{ env_collection.instance_name }}-efs-access] 16 | } 17 | 18 | resource "aws_security_group" "{{ env_collection.instance_name }}-efs-access" { 19 | name = "{{ env_collection.instance_name }}-efs-access" 20 | description = "Allow ecs/efs communication" 21 | vpc_id = {{ env_collection.vpc_resource_address }}.id 22 | 23 | ingress { 24 | from_port = 2049 25 | to_port = 2049 26 | protocol = "tcp" 27 | security_groups = [aws_security_group.{{ env_collection.instance_name }}_ecs.id] 28 | } 29 | 30 | depends_on = [{{ env_collection.vpc_resource_address }}] 31 | } 32 | {% set efs_mountpoint_status.created = True %} 33 | {% endif %} 34 | {% endif %} 35 | {% endfor %} 36 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/partials/environment/msg_queue_and_obj_storage.tf.j2: -------------------------------------------------------------------------------- 1 | {% for queue in environment.message_queue_resources %} 2 | {% if queue.settings.use_existing %} 3 | resource "aws_sqs_queue" "{{ queue.tf_safe_queue_name}}" { 4 | name = "{{ queue.queue_name }}" 5 | {% if queue.is_fifo %} 6 | fifo_queue = true 7 | {% endif %} 8 | } 9 | 10 | output "aws_sqs_queue_{{ queue.tf_safe_queue_name }}" { 11 | value = aws_sqs_queue.{{ queue.tf_safe_queue_name }}.arn 12 | } 13 | {% endif %} 14 | {% endfor %} 15 | 16 | {% for bucket in environment.object_storage_resources %} 17 | {% if not bucket.settings.use_existing %} 18 | resource "aws_s3_bucket" "{{ bucket.settings.bucket_name }}" { 19 | bucket = "{{ bucket.settings.bucket_name }}" 20 | 21 | {% if not environment.is_static %} 22 | force_destroy = true 23 | {% endif %} 24 | } 25 | 26 | {% if bucket.settings.cors %} 27 | resource "aws_s3_bucket_cors_configuration" "{{ bucket.settings.bucket_name }}" { 28 | bucket = aws_s3_bucket.{{ bucket.settings.bucket_name }}.id 29 | 30 | {% for cors_block in bucket.settings.cors %} 31 | cors_rule { 32 | allowed_headers = ["*"] 33 | allowed_methods = [{% for method in cors_block.allowed_methods %}"{{ method }}"{% if not loop.last %},{% endif %}{% endfor %}] 34 | allowed_origins = [{% for domain in cors_block.allowed_origins %}"{{ domain }}"{% if not loop.last %},{% endif %}{% endfor %}] 35 | max_age_seconds = 3000 36 | } 37 | {% endfor %} 38 | } 39 | {% endif %} 40 | 41 | resource "aws_s3_bucket_ownership_controls" "{{ bucket.settings.bucket_name }}" { 42 | bucket = aws_s3_bucket.{{ bucket.settings.bucket_name }}.id 43 | rule { 44 | object_ownership = "ObjectWriter" 45 | } 46 | } 47 | 48 | resource "aws_s3_bucket_acl" "{{ bucket.settings.bucket_name }}" { 49 | bucket = aws_s3_bucket.{{ bucket.settings.bucket_name }}.id 50 | acl = "private" 51 | depends_on = [ 52 | aws_s3_bucket_ownership_controls.{{ bucket.settings.bucket_name }}, 53 | ] 54 | } 55 | 56 | output "s3_bucket_{{ bucket.settings.bucket_name }}" { 57 | value = aws_s3_bucket.{{ bucket.settings.bucket_name }}.arn 58 | } 59 | {% endif %} 60 | {% endfor %} 61 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/partials/service/backend/alb_target_group.tf.j2: -------------------------------------------------------------------------------- 1 | resource "aws_alb_target_group" "{{ service.instance_name }}" { 2 | name = "{{ service.settings.target_group_name }}" 3 | port = 80 4 | protocol = "HTTP" 5 | vpc_id = {{ env_collection.vpc_resource_address }}.id 6 | target_type = "ip" 7 | 8 | health_check { 9 | path = "{{ service.settings.system.health_check }}" 10 | {% if service.settings.system.health_check_enabled %} 11 | matcher = "200-399" 12 | {% else %} 13 | matcher = "200-499" 14 | {% endif %} 15 | interval = 10 16 | healthy_threshold = 2 17 | unhealthy_threshold = 9 18 | } 19 | 20 | lifecycle { 21 | create_before_destroy = true 22 | } 23 | } -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/partials/service/backend/ecs_service.tf.j2: -------------------------------------------------------------------------------- 1 | resource "aws_ecs_service" "{{ service.instance_name }}" { 2 | name = "{{ service.instance_name }}" 3 | task_definition = "${aws_ecs_task_definition.{{ service.instance_name }}.family}" 4 | desired_count = 0 5 | 6 | cluster = "${aws_ecs_cluster.{{ env_collection.instance_name }}.id}" 7 | depends_on = [aws_iam_role_policy.{{ env_collection.instance_name }}_ecs_task, aws_alb_target_group.{{ service.instance_name }}] 8 | 9 | network_configuration { 10 | security_groups = "${concat([aws_security_group.{{ env_collection.instance_name }}_default.id, aws_security_group.{{ env_collection.instance_name }}_ecs.id], [aws_security_group.{{ env_collection.instance_name }}_db_access.id], [aws_security_group.{{ env_collection.instance_name }}_cache_access.id])}" 11 | subnets = concat( 12 | local.private_subnet_ids, 13 | length(local.private_subnet_ids) > 0 ? [] : aws_subnet.{{ env_collection.instance_name }}_private.*.id 14 | ) 15 | } 16 | 17 | load_balancer { 18 | target_group_arn = "${aws_alb_target_group.{{ service.instance_name }}.arn}" 19 | container_name = "{{ service.instance_name }}" 20 | container_port = "{{ service.port }}" 21 | } 22 | 23 | deployment_circuit_breaker { 24 | enable = true 25 | rollback = true 26 | } 27 | 28 | {% if service.environment.is_static %} 29 | launch_type = "FARGATE" 30 | {% else %} 31 | capacity_provider_strategy { 32 | capacity_provider = "FARGATE_SPOT" 33 | weight = 100 34 | } 35 | {% endif %} 36 | 37 | lifecycle { 38 | ignore_changes = [desired_count, task_definition] 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/partials/service/backend/scheduled_tasks.tf.j2: -------------------------------------------------------------------------------- 1 | {% for task in service.settings.scheduled_tasks %} 2 | resource "aws_ecs_task_definition" "{{ service.instance_name }}_{{ task.name }}" { 3 | family = "{{ service.instance_name }}_{{ task.name }}" 4 | container_definitions = jsonencode([ 5 | { 6 | name = "{{ service.instance_name }}_{{ task.name }}" 7 | image = "${aws_ecr_repository.{{ service.instance_name }}.repository_url}:latest" 8 | portMappings = [] 9 | entryPoint = ["sh"] 10 | command = [ 11 | "-c", 12 | "{{ task.command }}" 13 | ] 14 | } 15 | ]) 16 | requires_compatibilities = ["FARGATE"] 17 | network_mode = "awsvpc" 18 | cpu = "{{ task.system.cpu }}" 19 | memory = "{{ task.system.memory }}" 20 | execution_role_arn = "${aws_iam_role.{{ env_collection.instance_name }}_ecs_task.arn}" 21 | task_role_arn = "${aws_iam_role.{{ env_collection.instance_name }}_ecs_task.arn}" 22 | 23 | lifecycle { 24 | ignore_changes = all 25 | } 26 | } 27 | 28 | resource "aws_cloudwatch_event_rule" "{{ service.instance_name }}_{{ task.name }}" { 29 | name = "{{ service.instance_name[:39] }}-{{ task.name[:24] }}" 30 | schedule_expression = "cron({{ task.schedule }})" 31 | } 32 | 33 | resource "aws_cloudwatch_event_target" "{{ service.instance_name }}_{{ task.name }}" { 34 | rule = "${aws_cloudwatch_event_rule.{{ service.instance_name }}_{{ task.name }}.name}" 35 | target_id = "{{ service.instance_name }}-{{ task.name }}" 36 | arn = "${aws_ecs_cluster.{{ env_collection.instance_name }}.id}" 37 | role_arn = "${aws_iam_role.{{ env_collection.instance_name }}_ecs_events.arn}" 38 | 39 | ecs_target { 40 | task_count = 1 41 | task_definition_arn = "${aws_ecs_task_definition.{{ service.instance_name }}_{{ task.name }}.arn}" 42 | launch_type = "FARGATE" 43 | 44 | network_configuration { 45 | security_groups = "${concat([aws_security_group.{{ env_collection.instance_name }}_default.id, aws_security_group.{{ env_collection.instance_name }}_ecs.id], [aws_security_group.{{ env_collection.instance_name }}_db_access.id], [aws_security_group.{{ env_collection.instance_name }}_cache_access.id])}" 46 | subnets = concat( 47 | local.private_subnet_ids, 48 | length(local.private_subnet_ids) > 0 ? [] : aws_subnet.{{ env_collection.instance_name }}_private.*.id 49 | ) 50 | } 51 | } 52 | 53 | lifecycle { 54 | ignore_changes = [ecs_target] 55 | } 56 | } 57 | {% endfor %} 58 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/ecs/1/provision/partials/service_level_resources.tf.j2: -------------------------------------------------------------------------------- 1 | {% if env_collection.hosted_zone_ns_records %} 2 | 3 | resource "aws_ecr_repository" "{{ service.instance_name }}" { 4 | name = "{{ service.instance_name }}" 5 | force_delete = true 6 | } 7 | 8 | output "{{ service.instance_name }}_repository_url" { 9 | value = aws_ecr_repository.{{ service.instance_name }}.repository_url 10 | } 11 | 12 | {# {% if service.is_frontend %} 13 | {% include "partials/service/all_frontend.tf.j2" %} 14 | {% endif %}if service.is_frontend #} 15 | 16 | {% if service.is_backend %} 17 | resource "aws_cloudwatch_log_group" "{{ service.instance_name }}" { 18 | name = "{{ service.instance_name }}" 19 | tags = { 20 | Environment = "{{ env_collection.name }}" 21 | Application = "{{ app.name }}" 22 | } 23 | } 24 | {% endif %}{# if service.is_backend #} 25 | 26 | {% if environment.active_deployment %} 27 | {% if service.is_frontend %} 28 | {% include "partials/service/active_frontend.tf.j2" %} 29 | {% endif %}{# if service.is_frontend #} 30 | 31 | {% if service.is_backend %} 32 | {% include "partials/service/active_backend.tf.j2" %} 33 | {% endif %}{# if service.is_backend #} 34 | 35 | 36 | {% if service.is_serverless %} 37 | {% include "partials/collection/lambda/config_alb.tf.j2" %} 38 | {% endif %}{# if service.serverless_services #} 39 | 40 | {% endif %}{# if environment.active_deployment #} 41 | 42 | {% endif %}{# if hosted_zone_ns_records #} -------------------------------------------------------------------------------- /src/cnc/flavors/aws/lambda-lite/1/build/build_functions.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | send_{{ service.name }}_status_hook () { 4 | echo -e \\nSending build status webhook...\\n 5 | 6 | if [ $1 -eq 0 ]; then 7 | cnc_build_status=${2:-"success"} 8 | else 9 | cnc_build_status="failed" 10 | fi 11 | 12 | data='{ 13 | "token": "{{ builder.webhook_token }}", 14 | "status": "'"$cnc_build_status"'", 15 | "stage": "build", 16 | "service": "{{ service.name }}" 17 | }' 18 | echo "===== {{ service.name }} build status =====" 19 | echo "$data" | jq . 20 | echo "===== {{ service.name }} build status =====" 21 | 22 | which curl; curl_exists=$? 23 | if [ $curl_exists -ne 0 ]; then 24 | echo -e \\nWarning: Cannot find curl binary, skipping build status webhook...\\n 25 | else 26 | {% if builder.webhook_url %} 27 | webhook_status=$(curl -X PUT -H "Content-Type: application/json" \ 28 | -o /dev/null \ 29 | -d "$data" -w "%{http_code}" \ 30 | {{ builder.webhook_url }}) || true 31 | 32 | if [ "$webhook_status" != "200" ]; then 33 | echo -e \\nBuild status webhook failed. 34 | else 35 | echo -e \\nBuild status webhook succeeded. 36 | fi 37 | {% else %} 38 | echo -e \\nNo webhook URL provided. Skipping build status webhook... 39 | {% endif %} 40 | fi 41 | } 42 | 43 | build_{{ service.name }} () { 44 | echo "Verify Lambda function code..." 45 | 46 | source_dir="{{ service.build.context }}" 47 | 48 | if [ -d "$source_dir" ]; then 49 | echo "Source is a directory: $source_dir" 50 | elif [ -f "$source_dir" ]; then 51 | echo "Source is a file: $source_dir" 52 | else 53 | echo "Error: Source path is not valid: $source_dir" 54 | exit 1 55 | fi 56 | } 57 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/lambda-lite/1/build/main.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | trap 'send_{{ service.name }}_status_hook $?' EXIT 5 | 6 | if ! command -v bash >/dev/null 2>&1; then 7 | echo "Bash not found. Exiting." 8 | exit 1 9 | fi 10 | 11 | {% block install_commands %} 12 | source {{ builder.rendered_files_path }}/build-{{ service.name }}-functions.sh 13 | send_{{ service.name }}_status_hook 0 "working" 14 | build_{{ service.name }} 15 | {% endblock %} -------------------------------------------------------------------------------- /src/cnc/flavors/aws/lambda-lite/1/deploy/base.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | trap 'send_{{ service.name }}_status_hook $?' EXIT 5 | 6 | if ! command -v bash >/dev/null 2>&1; then 7 | echo "Bash not found. Exiting." 8 | exit 1 9 | fi 10 | 11 | {% block install_commands %} 12 | source {{ deployer.rendered_files_path }}/deploy-{{ service.name }}-functions.sh 13 | send_{{ service.name }}_status_hook 0 "working" 14 | deploy_{{ service.name }} 15 | {% endblock %} 16 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/lambda-lite/1/deploy/env_vars.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "Variables": { 3 | {%- for item in service.insecure_environment_items %} 4 | "{{ item.name }}": {{ item.value | tojson }}{%- if not loop.last or service.environment_secrets %},{%- endif %} 5 | {%- endfor %} 6 | 7 | {%- for secret in service.environment_secrets %} 8 | {% if service.settings.secrets_mode == 'plaintext' %} 9 | "{{ secret.name }}": {{ secret.value | tojson }}{%- if not loop.last %},{%- endif %} 10 | {%- elif service.settings.secrets_mode == 'arn' %} 11 | "{{ secret.name }}": "{{ secret.secret_id }}"{%- if not loop.last %},{%- endif %} 12 | {%- endif %} 13 | {%- endfor %} 14 | } 15 | } -------------------------------------------------------------------------------- /src/cnc/flavors/aws/lambda-lite/1/deploy/main.sh.j2: -------------------------------------------------------------------------------- 1 | {% extends "base.sh.j2" %} 2 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/lambda-lite/1/provision/base.tf.j2: -------------------------------------------------------------------------------- 1 | {% block state %} 2 | terraform { 3 | backend "local" { 4 | path = "{{ config_renderer.working_dir }}/.terraform/{{ config_renderer.tf_state_namespace }}.tfstate" 5 | } 6 | } 7 | {% endblock %} 8 | {% set lb_rules = namespace(priority=1) %} 9 | 10 | {% if not output_only %} 11 | 12 | {% block providers %} 13 | {% include "partials/collection/providers.tf.j2" %} 14 | {% endblock providers%} 15 | 16 | {% block environments %} 17 | {% include "partials/collection/environments.tf.j2" %} 18 | {% endblock environments %} 19 | 20 | 21 | {% block collection_level_resources %} 22 | {% if env_collection.database_resources %} 23 | {% block database_shared %} 24 | {% include "partials/collection/database/rds.tf.j2" %} 25 | {% endblock database_shared %} 26 | {% endif %} 27 | {% if env_collection.serverless_services %} 28 | 29 | {% block lambda_shared %} 30 | {% include "partials/collection/lambda/lambda.tf.j2" %} 31 | {% include "partials/collection/lambda/variables.tf.j2" %} 32 | {% include "partials/collection/lambda/outputs.tf.j2" %} 33 | {% endblock lambda_shared %} 34 | {% endif %} 35 | 36 | {% if env_collection.dynamodb_resources %} 37 | {% block dynamodb_shared %} 38 | {% include "partials/collection/dynamodb/dynamodb.tf.j2" %} 39 | {% include "partials/collection/dynamodb/variables.tf.j2" %} 40 | {% include "partials/collection/dynamodb/outputs.tf.j2" %} 41 | {% endblock dynamodb_shared %} 42 | {% endif %} 43 | {% endblock collection_level_resources %} 44 | 45 | {% endif %}{# {% if not output_only %} #} -------------------------------------------------------------------------------- /src/cnc/flavors/aws/lambda-lite/1/provision/main.tf.j2: -------------------------------------------------------------------------------- 1 | {% extends "base.tf.j2" %} 2 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/lambda-lite/1/provision/partials/collection/environments.tf.j2: -------------------------------------------------------------------------------- 1 | 2 | variable "account_id" { 3 | description = "Environment to be used on all the resources as identifier" 4 | type = string 5 | default = "{{ env_collection.account_id | default('null') }}" 6 | } 7 | 8 | variable "region" { 9 | description = "AWS region" 10 | type = string 11 | default = "{{ env_collection.region | default('us-east-1') }}" 12 | } 13 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/shared/.metadata: -------------------------------------------------------------------------------- 1 | provider_links_templates: 2 | serverless: https://{{ service.environment.collection.region }}.console.aws.amazon.com/lambda/home?region={{ service.environment.collection.region }}#/functions/{{ service.instance_name }}?tab=monitoring 3 | backend: "https://{{ service.environment.collection.region }}.console.aws.amazon.com/ecs/v2/clusters/{{ service.environment.collection.instance_name }}/services/{{ service.instance_name }}/deployments?region={{ service.environment.collection.region }}" 4 | worker: "https://{{ service.environment.collection.region }}.console.aws.amazon.com/ecs/v2/clusters/{{ service.environment.collection.instance_name }}/services/{{ service.instance_name }}_{{ worker.name }}/deployments?region={{ service.environment.collection.region }}" 5 | task: "https://{{ service.environment.collection.region }}.console.aws.amazon.com/ecs/v2/clusters/{{ service.environment.collection.instance_name }}/services/{{ service.instance_name }}_{{ task.name }}/deployments?region={{ service.environment.collection.region }}" 6 | database: "https://{{ service.environment.collection.region }}.console.aws.amazon.com/rds/home?region={{ service.environment.collection.region }}#database:id={{ service.instance_name }}" 7 | cache: "https://{{ service.environment.collection.region }}.console.aws.amazon.com/elasticache/home?region={{ service.environment.collection.region }}#/redis/{{ service.instance_name}}" 8 | frontend: "https://{{ service.environment.collection.region }}.console.aws.amazon.com/cloudfront/v3/home?region={{ service.environment.collection.region }}#/distributions" -------------------------------------------------------------------------------- /src/cnc/flavors/aws/shared/build/env_vars.sh.j2: -------------------------------------------------------------------------------- 1 | {%- for item in service.insecure_environment_items %} 2 | {{ item.name }}={{ item.value }} 3 | {%- endfor %} 4 | {%- for secret in service.environment_secrets %} 5 | {{ secret.name }}={{ secret.secret_id }} 6 | {%- endfor %} 7 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/shared/provision/lambda_function_payload/lambda_function.py: -------------------------------------------------------------------------------- 1 | def lambda_handler(event, context): 2 | return { 3 | "headers": {"Content-Type": "text/html"}, 4 | "statusCode": 200, 5 | "body": "Hello from Lambda!", 6 | } 7 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/shared/provision/partials/collection/dynamodb/outputs.tf.j2: -------------------------------------------------------------------------------- 1 | {% for environment in env_collection.active_environments %} 2 | {% for resource in environment.dynamodb_resources %} 3 | {% if resource.is_dynamodb %} 4 | 5 | output "{{ resource.instance_name }}_dynamodb_table_arn" { 6 | description = "ARN of the DynamoDB table" 7 | value = aws_dynamodb_table.{{ resource.instance_name }}.arn 8 | } 9 | 10 | output "{{ resource.instance_name }}_dynamodb_table_id" { 11 | description = "ID of the DynamoDB table" 12 | value = aws_dynamodb_table.{{ resource.instance_name }}.id 13 | } 14 | 15 | output "{{ resource.instance_name }}_dynamodb_table_billing_mode" { 16 | description = "Billing mode of the DynamoDB table" 17 | value = aws_dynamodb_table.{{ resource.instance_name }}.billing_mode 18 | } 19 | {% endif %} 20 | {% endfor %} 21 | {% endfor %} -------------------------------------------------------------------------------- /src/cnc/flavors/aws/shared/provision/partials/collection/lambda/outputs.tf.j2: -------------------------------------------------------------------------------- 1 | {% for environment in env_collection.active_environments %} 2 | {% for service in environment.services %} 3 | {% if service.is_serverless %} 4 | 5 | output "{{ service.instance_name }}_lambda_function_name" { 6 | description = "The name of the Lambda Function" 7 | value = "${aws_lambda_function.{{ service.instance_name }}.function_name}" 8 | } 9 | 10 | output "{{ service.instance_name }}_lambda_function_url" { 11 | description = "The URL of the Lambda Function URL" 12 | value = "${aws_lambda_function_url.{{ service.instance_name }}.function_url}" 13 | } 14 | 15 | output "{{ service.instance_name }}_lambda_function_url_id" { 16 | description = "The Lambda Function URL generated id" 17 | value = "${aws_lambda_function_url.{{ service.instance_name }}.url_id}" 18 | } 19 | 20 | output "{{ service.instance_name }}_lambda_function_version" { 21 | description = "Latest published version of Lambda Function" 22 | value = "${aws_lambda_function.{{ service.instance_name }}.version}" 23 | } 24 | 25 | output "{{ service.instance_name }}_lambda_role_arn" { 26 | description = "The ARN of the IAM role created for the Lambda Function" 27 | value = "${aws_iam_role.{{ service.instance_name }}.arn}" 28 | } 29 | 30 | output "{{ service.instance_name }}_lambda_role_name" { 31 | description = "The name of the IAM role created for the Lambda Function" 32 | value = "${aws_iam_role.{{ service.instance_name }}.name}" 33 | } 34 | 35 | {% endif%} 36 | {% endfor %} 37 | {% endfor %} -------------------------------------------------------------------------------- /src/cnc/flavors/aws/shared/provision/partials/collection/lambda/variables.tf.j2: -------------------------------------------------------------------------------- 1 | {% for environment in env_collection.active_environments %} 2 | {% for service in environment.services %} 3 | {% if service.is_serverless %} 4 | 5 | variable "{{ service.instance_name }}_aws_resources_permission_lambda" { 6 | description = "List of AWS resources that the Lambda function needs to access" 7 | type = list(string) 8 | default = [ "logs", "cloudwatch", "ec2", "dynamodb", "s3"] 9 | } 10 | 11 | variable "{{ service.instance_name }}_handler" { 12 | description = "Lambda function handler | e.g Format: filename.lambda_handler" 13 | type = string 14 | default = "{{ service.settings.handler | default('lambda_function.lambda_handler') }}" 15 | } 16 | 17 | variable "{{ service.instance_name }}_runtime" { 18 | description = "Lambda function runtime" 19 | type = string 20 | default = "{{ service.settings.runtime | default('python3.12') }}" 21 | } 22 | 23 | {% endif %} 24 | {% endfor %} 25 | {% endfor %} -------------------------------------------------------------------------------- /src/cnc/flavors/aws/shared/provision/partials/collection/providers.tf.j2: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "{{ env_collection.region }}" 3 | allowed_account_ids = ["{{ env_collection.account_id }}"] 4 | 5 | default_tags { 6 | tags = { 7 | Environment = "{{ env_collection.name }}" 8 | ManagedBy = "cnc" 9 | Application = "{{ app.name }}" 10 | } 11 | } 12 | } 13 | 14 | {# To be used with lambda edge functions as they must be locked to us-east-1 #} 15 | provider "aws" { 16 | alias = "us_east_1" 17 | region = "us-east-1" 18 | allowed_account_ids = ["{{ env_collection.account_id }}"] 19 | 20 | default_tags { 21 | tags = { 22 | Environment = "{{ env_collection.name }}" 23 | ManagedBy = "cnc" 24 | Application = "{{ app.name }}" 25 | } 26 | } 27 | } 28 | 29 | terraform { 30 | required_providers { 31 | aws = { 32 | source = "hashicorp/aws" 33 | version = "~> 5.25" 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /src/cnc/flavors/aws/shared/toolbox/Dockerfile.j2: -------------------------------------------------------------------------------- 1 | FROM {{ service.image_for_tag(toolbox.tag_for_service(service.name)) }} 2 | 3 | ARG AWS_DEFAULT_REGION 4 | ENV AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 5 | 6 | ARG AWS_REGION 7 | ENV AWS_REGION=$AWS_REGION 8 | 9 | ARG AWS_ACCESS_KEY_ID 10 | ENV AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID 11 | 12 | ARG AWS_SECRET_ACCESS_KEY 13 | ENV AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY 14 | 15 | ARG AWS_SESSION_TOKEN 16 | ENV AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN 17 | 18 | COPY .aws /_cnc_temp/.aws 19 | RUN cp -r /_cnc_temp/.aws ~/.aws 20 | -------------------------------------------------------------------------------- /src/cnc/flavors/aws/shared/toolbox/main.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TOOLBOX_ACTIVE_TEMP_FILEPATH=/tmp/cnc_toolbox_active 4 | touch $TOOLBOX_ACTIVE_TEMP_FILEPATH 5 | 6 | # This kills all processes spawned by this script on exit 7 | trap cleanup_toolbox EXIT 8 | 9 | # Echo caller identity 10 | echo -e "\nYou are currently authenticated as: $(aws sts get-caller-identity)" 11 | 12 | {%- if environment.database_resources or environment.cache_resources %} 13 | cleanup_toolbox () { 14 | echo -e "\nCleaning up port-forwarding sessions..." 15 | rm $TOOLBOX_ACTIVE_TEMP_FILEPATH 16 | 17 | # Use pkill instead of pgrep and kill 18 | pkill -P $port_forwarding_parent_process_id 19 | 20 | {%- for resource in environment.database_resources + environment.cache_resources %} 21 | if [ -f /tmp/cnc_ssh_output_{{ loop.index }} ] 22 | then 23 | rm /tmp/cnc_ssh_output_{{ loop.index }} 24 | fi 25 | {%- endfor %} 26 | } 27 | 28 | {# Start ssm sessions #} 29 | start_ssm_sessions () { 30 | PID_FALLBACK="none" 31 | while [ -f $TOOLBOX_ACTIVE_TEMP_FILEPATH ] 32 | do 33 | {% for resource in environment.database_resources + environment.cache_resources %} 34 | if ! kill -0 ${SSM_PID_{{ loop.index }}:-$PID_FALLBACK} 2>/dev/null 35 | then 36 | SSM_RETRY_COUNT_{{ loop.index }}=${SSM_RETRY_COUNT_{{ loop.index }}:-0} 37 | if [ $SSM_RETRY_COUNT_{{ loop.index }} -eq 0 ] 38 | then 39 | echo -e "\nStarting ssm port-forwarding session for {{ resource.name }}..." 40 | fi 41 | 42 | aws ssm start-session --target {{ bastion_instance_id }} \ 43 | --document-name AWS-StartPortForwardingSessionToRemoteHost \ 44 | --parameters '{{ resource.settings.toolbox_ssh_port_mapping }}' > /tmp/cnc_ssh_output_{{ loop.index }} 2>&1 & 45 | SSM_PID_{{ loop.index }}=$! 46 | 47 | if [ $SSM_RETRY_COUNT_{{ loop.index }} -gt 3 ] 48 | then 49 | cmd_output=$(/dev/null 2>&1; then 7 | echo "Bash not found. Exiting." 8 | exit 1 9 | fi 10 | 11 | {% block install_commands %} 12 | source {{ builder.rendered_files_path }}/build-{{ service.name }}-functions.sh 13 | gcloud auth configure-docker {{ environment.collection.region }}-docker.pkg.dev 14 | send_{{ service.name }}_status_hook 0 "working" 15 | {% endblock %} 16 | 17 | {% block pre_build_commands %} 18 | {% endblock %} 19 | 20 | {%- block render_templates %} 21 | {%- if service.is_frontend %} 22 | {{ render_template("Dockerfile.run.j2", "Dockerfile.{}".format(service.name)) }} 23 | {{ render_template("nginx.conf.j2", "{}-nginx.conf".format(service.name)) }} 24 | {%- endif %} 25 | {%- endblock %} 26 | 27 | {% block build_commands %} 28 | 29 | log_{{ service.name }}_variables 30 | 31 | {%- if service.image %} 32 | verify_{{ service.name }}_image_exists 33 | {%- else %} 34 | build_{{ service.name }}_image 35 | 36 | {%- if service.build %} 37 | push_{{ service.name }}_image_tags 38 | {%- endif %} 39 | 40 | 41 | {% if service.is_frontend %} 42 | build_{{ service.name }}_assets 43 | build_{{ service.name }}_run_image 44 | {% endif %} 45 | 46 | {% endif %} 47 | 48 | {% endblock build_commands %} 49 | 50 | {% block post_build_commands %} 51 | {% endblock %} -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/build/env_vars.sh.j2: -------------------------------------------------------------------------------- 1 | {%- for item in service.insecure_environment_items %} 2 | {{ item.name }}={{ item.value }} 3 | {%- endfor %} 4 | {%- for secret in service.environment_secrets %} 5 | {{ secret.name }}={{ secret.secret_id }} 6 | {%- endfor %} 7 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/deploy/all-access-policy.yml.j2: -------------------------------------------------------------------------------- 1 | bindings: 2 | - members: 3 | - allUsers 4 | role: roles/run.invoker -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/deploy/k8s/scheduled_tasks.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: "{{ service.instance_name }}" 6 | --- 7 | apiVersion: v1 8 | kind: ServiceAccount 9 | metadata: 10 | namespace: "{{ service.instance_name }}" 11 | name: "{{ service.instance_name }}" 12 | --- 13 | apiVersion: rbac.authorization.k8s.io/v1 14 | kind: Role 15 | metadata: 16 | namespace: "{{ service.instance_name }}" 17 | name: "{{ service.instance_name }}" 18 | rules: 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - pods 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | --- 28 | apiVersion: rbac.authorization.k8s.io/v1 29 | kind: RoleBinding 30 | metadata: 31 | namespace: "{{ service.instance_name }}" 32 | name: "{{ service.instance_name }}" 33 | roleRef: 34 | apiGroup: rbac.authorization.k8s.io 35 | kind: Role 36 | name: "{{ service.instance_name }}" 37 | subjects: 38 | - kind: ServiceAccount 39 | name: "{{ service.instance_name }}" 40 | namespace: "{{ service.instance_name }}" 41 | --- 42 | {% for task in service.settings.scheduled_tasks %} 43 | apiVersion: batch/v1 44 | kind: CronJob 45 | metadata: 46 | namespace: "{{ service.instance_name }}" 47 | name: "{{ service.name }}-{{ task.name }}" 48 | spec: 49 | # https://cloud.google.com/kubernetes-engine/docs/how-to/cronjobs#schedule 50 | # "*/1 * * * *" 51 | schedule: "{{ task.schedule }}" 52 | # todo, allow user to set 53 | concurrencyPolicy: Replace 54 | startingDeadlineSeconds: 100 55 | # todo, allow user to set 56 | # suspend: false 57 | successfulJobsHistoryLimit: 5 58 | failedJobsHistoryLimit: 10 59 | jobTemplate: 60 | spec: 61 | backoffLimit: 0 62 | template: 63 | spec: 64 | terminationGracePeriodSeconds: 1800 65 | affinity: 66 | nodeAffinity: 67 | preferredDuringSchedulingIgnoredDuringExecution: 68 | - weight: 1 69 | preference: 70 | matchExpressions: 71 | - key: cloud.google.com/gke-spot 72 | operator: {% if service.environment.is_static %}NotIn{% else %}In{% endif %} 73 | values: 74 | - "true" 75 | restartPolicy: "Never" 76 | serviceAccountName: "{{ service.instance_name }}" 77 | securityContext: 78 | runAsUser: 3298 79 | runAsGroup: 3298 80 | fsGroup: 3298 81 | containers: 82 | {% filter indent(width=20) %} 83 | {% with worker=None %} 84 | {% include "k8s/service_containers/backend-worker.yml" %} 85 | {% endwith %} 86 | {% endfilter %} 87 | --- 88 | {% endfor %} 89 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/deploy/k8s/service_containers/backend-worker.yml: -------------------------------------------------------------------------------- 1 | - image: {{ service.image_for_tag(deployer.tag_for_service(service.name)) }} 2 | name: {{ service.name }}-{% if worker %}{{ worker.name }}{% elif task %}{{ task.name }}{% endif %} 3 | resources: 4 | requests: 5 | {% if worker %} 6 | cpu: "{{ worker.system.gke_autopilot_cpu }}" 7 | memory: "{{ worker.system.gke_autopilot_memory }}" 8 | {% elif task %} 9 | cpu: "{{ task.system.gke_autopilot_cpu }}" 10 | memory: "{{ task.system.gke_autopilot_memory }}" 11 | {% endif %} 12 | ephemeral-storage: "5Gi" 13 | limits: 14 | {% if worker %} 15 | cpu: "{{ worker.system.gke_autopilot_cpu }}" 16 | memory: "{{ worker.system.gke_autopilot_memory }}" 17 | {% elif task %} 18 | cpu: "{{ task.system.gke_autopilot_cpu }}" 19 | memory: "{{ task.system.gke_autopilot_memory }}" 20 | {% endif %} 21 | ephemeral-storage: "5Gi" 22 | env: 23 | - name: CNC_PROVISIONED 24 | value: "1" 25 | {% for item in service.environment_items %} 26 | - name: "{{ item.name }}" 27 | valueFrom: 28 | secretKeyRef: 29 | name: "{{ item.instance_name }}" 30 | key: "value" 31 | {% endfor %} 32 | command: 33 | - "bash" 34 | - "-c" 35 | - | 36 | sleep 20 37 | {% if worker %} 38 | {{ worker.joined_command }} 39 | {% elif task %} 40 | {{ task.joined_command }} 41 | {% endif %} 42 | 43 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/deploy/k8s/workers.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: {{ service.instance_name }} 6 | --- 7 | apiVersion: v1 8 | kind: ServiceAccount 9 | metadata: 10 | namespace: {{ service.instance_name }} 11 | name: {{ service.instance_name }} 12 | {% for worker in service.settings.workers %} 13 | --- 14 | apiVersion: apps/v1 15 | kind: Deployment 16 | metadata: 17 | name: {{ service.name }}-{{ worker.name }} 18 | namespace: {{ service.instance_name }} 19 | spec: 20 | replicas: {{ worker.replicas }} 21 | revisionHistoryLimit: 0 22 | selector: 23 | matchLabels: 24 | app: {{ service.name }}-{{ worker.name }} 25 | template: 26 | metadata: 27 | labels: 28 | app: {{ service.name }}-{{ worker.name }} 29 | {% if worker.is_ext_duration_enabled %} 30 | duration: extended 31 | annotations: 32 | cluster-autoscaler.kubernetes.io/safe-to-evict: "false" 33 | {% endif %} 34 | spec: 35 | terminationGracePeriodSeconds: 10800 36 | affinity: 37 | nodeAffinity: 38 | requiredDuringSchedulingIgnoredDuringExecution: 39 | nodeSelectorTerms: 40 | - matchExpressions: 41 | - key: cloud.google.com/gke-spot 42 | operator: {% if service.environment.is_static %}NotIn{% else %}In{% endif %} 43 | values: 44 | - "true" 45 | serviceAccountName: {{ service.instance_name }} 46 | containers: 47 | {% filter indent(width=16) %} 48 | {% include "k8s/service_containers/backend-worker.yml" %} 49 | {% endfilter %} 50 | 51 | {% endfor %} -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/deploy/run-container.yml.j2: -------------------------------------------------------------------------------- 1 | {%- set is_job = is_job | default(False) %} 2 | spec: 3 | template: 4 | metadata: 5 | annotations: 6 | {% if service.is_backend %} 7 | run.googleapis.com/vpc-access-connector: {{ service.environment.collection.instance_name[:19].strip("-") }} 8 | run.googleapis.com/vpc-access-egress: all 9 | {% endif %} 10 | {%- if not is_job %} 11 | autoscaling.knative.dev/minScale: "{{ service.min_scale }}" 12 | {% endif %} 13 | run.googleapis.com/execution-environment: "gen2" 14 | {%- if is_job %} 15 | spec: 16 | template: 17 | {%- endif %} 18 | {% filter indent(4 if is_job else 0) %} 19 | spec: 20 | timeoutSeconds: 1800 21 | serviceAccountName: {{ service.environment.collection.service_identity_email }} 22 | {%- if not is_job %} 23 | containerConcurrency: 80 24 | {%- endif %} 25 | {%- if is_job %} 26 | maxRetries: 0 27 | {% endif %} 28 | containers: 29 | {% if service.is_frontend %} 30 | - image: {{ service.image_for_tag(deployer.tag_for_service(service.name), run=True) }} 31 | {% else %} 32 | - image: {{ service.image_for_tag(deployer.tag_for_service(service.name)) }} 33 | {% endif %} 34 | {% if service.is_backend and service.command %} 35 | {%- set svc_command = svc_command | default(service.joined_command) %} 36 | command: 37 | - "sh" 38 | - "-c" 39 | {%- if not is_job %} 40 | - | 41 | {{ svc_command }} 42 | {% endif %} 43 | {% endif %} 44 | env: 45 | - name: CNC_DEPLOY_TIMESTAMP 46 | value: "{{ deployer.current_timestamp }}" 47 | - name: CNC_PROVISIONED 48 | value: "1" 49 | {% for variable in service.insecure_environment_items %} 50 | {% if variable.name != "PORT" %} 51 | - name: {{ variable.name }} 52 | value: {{ variable.value | tojson }} 53 | {% endif %} 54 | {% endfor %} 55 | {% for secret in service.environment_secrets %} 56 | - name: {{ secret.name }} 57 | valueFrom: 58 | secretKeyRef: 59 | key: "latest" 60 | name: {{ secret.secret_id }} 61 | {%- endfor %} 62 | resources: 63 | limits: 64 | cpu: "{{ service.deploy.resources.limits.cpu }}" 65 | memory: "{{ service.deploy.resources.limits.memory }}" 66 | {% if service.is_frontend %} 67 | ports: 68 | - containerPort: 80 69 | {% endif %} 70 | {% endfilter %} 71 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/deploy/run-job.yml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: run.googleapis.com/v1 2 | kind: Job 3 | metadata: 4 | name: {{ service.instance_name }} 5 | 6 | {% with is_job=True %} 7 | {% include "run-container.yml.j2" %} 8 | {% endwith %} 9 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/deploy/run-svc.yml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: serving.knative.dev/v1 2 | kind: Service 3 | metadata: 4 | name: {{ service.instance_name }} 5 | labels: 6 | cloud.googleapis.com/location: {{ service.environment.collection.region }} 7 | annotations: 8 | run.googleapis.com/launch-stage: BETA 9 | 10 | {% include "run-container.yml.j2" %} -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/provision/partials/collection/bastion_instance.tf.j2: -------------------------------------------------------------------------------- 1 | {% if env_collection.has_active_deployments %} 2 | resource "google_compute_instance" "{{ env_collection.instance_name }}-bastion" { 3 | name = "{{ env_collection.instance_name }}-bastion" 4 | machine_type = "{{ env_collection.bastion_instance_type }}" 5 | zone = "{{ env_collection.region }}-b" 6 | allow_stopping_for_update = true 7 | deletion_protection = false 8 | 9 | tags = ["bastion"] 10 | 11 | boot_disk { 12 | initialize_params { 13 | image = "debian-cloud/debian-11" 14 | } 15 | } 16 | 17 | network_interface { 18 | network = google_compute_network.app-network.id 19 | subnetwork = "{{ env_collection.instance_name }}" 20 | 21 | access_config { 22 | // Ephemeral public IP 23 | } 24 | } 25 | 26 | depends_on = [google_compute_network.app-network] 27 | } 28 | 29 | resource "google_project_iam_member" "project-app-ssh-bastion" { 30 | project = "{{ env_collection.account_id }}" 31 | role = "roles/compute.instanceAdmin.v1" 32 | member = "serviceAccount:${google_service_account.{{ env_collection.instance_name }}.email}" 33 | depends_on = [google_project_service.iam, google_compute_instance.{{ env_collection.instance_name }}-bastion] 34 | } 35 | 36 | resource "google_compute_firewall" "rules" { 37 | project = "{{ env_collection.account_id }}" 38 | name = "bastion-{{ env_collection.instance_name }}" 39 | network = google_compute_network.app-network.id 40 | description = "Allow ssh to bastion for {{ env_collection.instance_name }}" 41 | 42 | allow { 43 | protocol = "tcp" 44 | ports = ["22"] 45 | } 46 | 47 | source_ranges = ["0.0.0.0/0"] 48 | target_tags = ["bastion"] 49 | } 50 | 51 | {% endif %}{# if env_collection.has_active_deployments #} 52 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/provision/partials/collection/providers.tf.j2: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | google = { 4 | source = "hashicorp/google" 5 | version = "~> 5.0" 6 | } 7 | 8 | google-beta = { 9 | source = "hashicorp/google-beta" 10 | version = "~> 5.0" 11 | } 12 | } 13 | } 14 | 15 | provider "google" { 16 | project = "{{ env_collection.account_id }}" 17 | region = "{{ env_collection.region }}" 18 | 19 | default_labels = { 20 | environment = "{{ env_collection.name.replace("-", "") }}" 21 | managed_by = "cnc" 22 | application = "{{ app.name }}" 23 | } 24 | } 25 | 26 | provider "google-beta" { 27 | project = "{{ env_collection.account_id }}" 28 | region = "{{ env_collection.region }}" 29 | 30 | default_labels = { 31 | environment = "{{ env_collection.name.replace("-", "") }}" 32 | managed_by = "cnc" 33 | application = "{{ app.name }}" 34 | } 35 | } 36 | 37 | data "google_project" "project" { 38 | project_id = "{{ env_collection.account_id }}" 39 | } 40 | 41 | {% if env_collection.needs_k8s %} 42 | data "google_client_config" "{{ env_collection.instance_name }}" {} 43 | 44 | provider "kubernetes" { 45 | host = "https://${google_container_cluster.{{ env_collection.instance_name }}.endpoint}" 46 | token = data.google_client_config.{{ env_collection.instance_name }}.access_token 47 | cluster_ca_certificate = base64decode( 48 | google_container_cluster.{{ env_collection.instance_name }}.master_auth[0].cluster_ca_certificate 49 | ) 50 | } 51 | {% endif %} 52 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/provision/partials/environment/object_storage_resources.tf.j2: -------------------------------------------------------------------------------- 1 | {%- if environment.active_deployment %} 2 | {%- for bucket in environment.object_storage_resources %} 3 | {%- if not bucket.settings.use_existing %} 4 | resource "google_storage_bucket" "{{ bucket.settings.bucket_name }}" { 5 | name = "{{ bucket.settings.bucket_name }}" 6 | location = "US" 7 | 8 | uniform_bucket_level_access = true 9 | 10 | {% for cors_block in bucket.settings.cors %} 11 | cors { 12 | origin = [ 13 | {% for domain in cors_block.allowed_origins %}"https://{{ domain }}"{% if not loop.last %},{% endif %}{% endfor %} 14 | ] 15 | method = [{% for method in cors_block.allowed_methods %}"{{ method }}"{% if not loop.last %},{% endif %}{% endfor %}] 16 | response_header = ["*"] 17 | max_age_seconds = 3600 18 | } 19 | {% endfor %}{# for cors_block in bucket.cors #} 20 | } 21 | 22 | resource "google_storage_bucket_iam_binding" "{{ bucket.settings.bucket_name }}" { 23 | bucket = google_storage_bucket.{{ bucket.settings.bucket_name }}.name 24 | role = "roles/storage.objectAdmin" 25 | members = [ 26 | "serviceAccount:${google_service_account.{{ env_collection.instance_name }}.email}" 27 | ] 28 | } 29 | 30 | output "gcs_bucket_{{ bucket.settings.bucket_name }}" { 31 | value = google_storage_bucket.{{ bucket.settings.bucket_name }}.url 32 | } 33 | {%- endif %}{# if bucket.settings.use_existing #} 34 | {%- endfor %}{# for bucket in environment.object_storage_resources #} 35 | {%- endif %} 36 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/provision/partials/environment/resource_managed_secrets.tf.j2: -------------------------------------------------------------------------------- 1 | {%- if environment.active_deployment %} 2 | {%- for resource in environment.config.resources %} 3 | {%- for k, v in resource.settings.managed_secret_values_for_tf.items() %} 4 | resource "google_secret_manager_secret" "{{ k }}" { 5 | secret_id = "{{ k }}" 6 | 7 | replication { 8 | auto {} 9 | } 10 | } 11 | 12 | resource "google_secret_manager_secret_version" "{{ k }}" { 13 | secret = "${google_secret_manager_secret.{{ k }}.id}" 14 | secret_data = "{{ v }}" 15 | } 16 | {%- endfor %} 17 | {%- endfor %} 18 | {%- endif %} 19 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/provision/partials/service/cache.tf.j2: -------------------------------------------------------------------------------- 1 | {%- if service.environment.active_deployment and service.settings.is_cache %} 2 | resource "google_redis_instance" "{{ service.instance_name }}" { 3 | name = "{{ service.instance_name }}" 4 | {% if environment.is_production %} 5 | tier = "STANDARD_HA" 6 | {% else %} 7 | tier = "BASIC" 8 | {% endif %} 9 | memory_size_gb = 1 10 | region = "{{ env_collection.region }}" 11 | 12 | {% if environment.is_production %} 13 | lifecycle { 14 | ignore_changes = all 15 | } 16 | {% endif %} 17 | 18 | authorized_network = google_compute_network.app-network.id 19 | connect_mode = "PRIVATE_SERVICE_ACCESS" 20 | 21 | redis_version = "{{ service.settings.provider_version }}" 22 | display_name = "{{ service.instance_name }} Redis Instance" 23 | 24 | depends_on = [ 25 | {%- if vpc_enabled %} 26 | google_service_networking_connection.private_service_connection, 27 | {%- endif %} 28 | google_project_service.redis 29 | ] 30 | } 31 | 32 | output "redis_ip_{{ service.instance_name }}" { 33 | value = google_redis_instance.{{ service.instance_name }}.host 34 | } 35 | 36 | output "redis_port_{{ service.instance_name }}" { 37 | value = google_redis_instance.{{ service.instance_name }}.port 38 | } 39 | {%- endif %} 40 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/provision/partials/service/cloud_run_service.tf.j2: -------------------------------------------------------------------------------- 1 | {%- if service.environment.active_deployment and service.settings.is_web %} 2 | resource "google_cloud_run_service" "{{ service.instance_name }}" { 3 | project = "{{ env_collection.account_id }}" 4 | name = "{{ service.instance_name }}" 5 | location = "{{ env_collection.region }}" 6 | 7 | template { 8 | spec { 9 | containers { 10 | image = "us-docker.pkg.dev/cloudrun/container/hello" 11 | } 12 | } 13 | } 14 | 15 | traffic { 16 | percent = 100 17 | latest_revision = true 18 | } 19 | 20 | lifecycle { 21 | ignore_changes = all 22 | } 23 | 24 | depends_on = [ 25 | {%- if vpc_enabled %} 26 | google_service_networking_connection.private_service_connection, 27 | {%- endif %} 28 | google_project_service.run 29 | ] 30 | 31 | timeouts { 32 | create = "2m" 33 | } 34 | } 35 | 36 | output "{{ service.instance_name }}_cloud_run_url" { 37 | value = google_cloud_run_service.{{ service.instance_name }}.status.0.url 38 | } 39 | 40 | output "{{ service.instance_name }}_repository_url" { 41 | value = join("", [ 42 | "{{ env_collection.region }}-docker.pkg.dev/", 43 | "{{ env_collection.account_id }}/{{ env_collection.instance_name }}/", 44 | "{{ service.gcr_image_name }}", 45 | ]) 46 | } 47 | 48 | {%- if service.is_backend %} 49 | resource "google_cloud_run_v2_job" "{{ service.instance_name }}" { 50 | project = "{{ env_collection.account_id }}" 51 | name = "{{ service.instance_name }}" 52 | location = "{{ env_collection.region }}" 53 | 54 | lifecycle { 55 | ignore_changes = all 56 | } 57 | 58 | template { 59 | template { 60 | containers { 61 | image = "us-docker.pkg.dev/cloudrun/container/hello" 62 | } 63 | } 64 | } 65 | } 66 | {%- endif %} 67 | {%- endif %} 68 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/provision/partials/service/cloud_run_service_networking.tf.j2: -------------------------------------------------------------------------------- 1 | {%- if service.environment.active_deployment and service.settings.is_web %} 2 | resource "google_compute_region_network_endpoint_group" "{{ service.instance_name }}" { 3 | provider = google-beta 4 | name = "{{ service.instance_name }}" 5 | network_endpoint_type = "SERVERLESS" 6 | region = "{{ env_collection.region }}" 7 | 8 | cloud_run { 9 | service = google_cloud_run_service.{{ service.instance_name }}.name 10 | } 11 | 12 | depends_on = [google_project_service.compute, google_cloud_run_service.{{ service.instance_name }}] 13 | } 14 | 15 | resource "google_compute_backend_service" "{{ service.instance_name }}" { 16 | name = "{{ service.instance_name }}" 17 | 18 | protocol = "HTTP" 19 | port_name = "http" 20 | timeout_sec = 30 21 | 22 | {% if service.settings.cdn.enabled %} 23 | enable_cdn = true 24 | compression_mode = "AUTOMATIC" 25 | {% endif %} 26 | {% if service.is_frontend %} 27 | log_config { 28 | enable = true 29 | sample_rate = "1.0" 30 | } 31 | {% endif %} 32 | 33 | backend { 34 | group = google_compute_region_network_endpoint_group.{{ service.instance_name }}.id 35 | } 36 | 37 | {% if service.settings.custom_request_headers %} 38 | custom_request_headers = [{% for header in service.settings.custom_request_headers %}"{{ header.name }}:{{ header.value }}"{% if not loop.last %}, {% endif %}{% endfor %}] 39 | {% endif %} 40 | {% if service.settings.custom_response_headers %} 41 | custom_response_headers = [{% for header in service.settings.custom_response_headers %}"{{ header.name }}{{ header.value }}"{% if not loop.last %}, {% endif %}{% endfor %}] 42 | {% endif %} 43 | 44 | depends_on = [ 45 | google_project_service.compute, 46 | google_compute_region_network_endpoint_group.{{ service.instance_name }} 47 | ] 48 | } 49 | {%- endif %} 50 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/toolbox/Dockerfile.j2: -------------------------------------------------------------------------------- 1 | FROM {{ service.image_for_tag(toolbox.tag_for_service(service.name)) }} 2 | 3 | COPY .config/gcloud /_cnc_temp/gcloud 4 | RUN mkdir -p ~/.config/gcloud && cp -r /_cnc_temp/gcloud/* ~/.config/gcloud || echo "Unable to copy gcloud config... skipping" 5 | 6 | # GOOGLE_APPLICATION_CREDENTIALS will be set to this filepath 7 | # when appropriate. see: main.sh.j2 8 | COPY service_account.json /_cnc_temp/service_account.json 9 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/toolbox/partials/proxy_only.sh.j2: -------------------------------------------------------------------------------- 1 | sleep 5 2 | echo -e "\nToolbox started in proxy only mode.\n" 3 | 4 | echo -e "\nThe following environment variables are \ 5 | normally injected into your service container. They can be used to establish \ 6 | connections to your GCP resources from your local machine.\n" 7 | 8 | {%- for variable in toolbox.environment_items %} 9 | echo -e {{ variable.name }}={{ shlex.quote(variable.value) }} 10 | {%- endfor %} 11 | 12 | echo -e "\n Press Ctrl+C to exit. (all port-forwarding sessions will be closed)" 13 | 14 | # Wait indefinitely 15 | while true; do sleep 1; done 16 | -------------------------------------------------------------------------------- /src/cnc/flavors/gcp/shared/toolbox/partials/service_container.sh.j2: -------------------------------------------------------------------------------- 1 | {{ render_template("Dockerfile.j2", "Dockerfile") }} 2 | # copy config from default location 3 | if [ ! -d "{{ toolbox.rendered_files_path }}/.config" ] 4 | then 5 | mkdir -p "{{ toolbox.rendered_files_path }}/.config" 6 | fi 7 | 8 | GCP_DEFAULT_CONFIG_DIR=${CLOUDSDK_CONFIG:-$HOME/.config/gcloud} 9 | if [ -d "$GCP_DEFAULT_CONFIG_DIR" ] 10 | then 11 | cp -R "$GCP_DEFAULT_CONFIG_DIR" "{{ toolbox.rendered_files_path }}/.config/gcloud" 12 | else 13 | mkdir -p "{{ toolbox.rendered_files_path }}/.config/gcloud" 14 | fi 15 | 16 | # If google credentials file is set then we copy 17 | # to known location and set correct GOOGLE_APPLICATION_CREDENTIALS path 18 | # for the toolbox Dockerfile 19 | if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ] 20 | then 21 | if [ -e "$GOOGLE_APPLICATION_CREDENTIALS" ] 22 | then 23 | cp "$GOOGLE_APPLICATION_CREDENTIALS" "{{ toolbox.rendered_files_path }}/service_account.json" 24 | TOOLBOX_GOOGLE_APPLICATION_CREDENTIALS="/_cnc_temp/service_account.json" 25 | fi 26 | fi 27 | touch "{{ toolbox.rendered_files_path }}/service_account.json" 28 | 29 | docker pull {{ service.image_for_tag(toolbox.tag_for_service(service.name)) }} 30 | 31 | docker build -t {{ service.instance_name }}_toolbox \ 32 | -f "{{ toolbox.rendered_files_path }}/Dockerfile" "{{ toolbox.rendered_files_path }}" 33 | 34 | {%- if command %} 35 | {# Allow time for port forwarding processes to start #} 36 | sleep ${CNC_TOOLBOX_CMD_DELAY_SECONDS:-15} 37 | {%- endif %} 38 | 39 | docker_auth_var="AUTH_AUTO_DETECTED=false" 40 | if [ -n "$TOOLBOX_GOOGLE_APPLICATION_CREDENTIALS" ] 41 | then 42 | docker_auth_var="GOOGLE_APPLICATION_CREDENTIALS=$TOOLBOX_GOOGLE_APPLICATION_CREDENTIALS" 43 | elif [ -n "$GOOGLE_AUTH_TOKEN" ] 44 | then 45 | docker_auth_var="CLOUDSDK_AUTH_ACCESS_TOKEN=$GOOGLE_AUTH_TOKEN" 46 | elif [ -n "$CLOUDSDK_AUTH_ACCESS_TOKEN" ] 47 | then 48 | docker_auth_var="CLOUDSDK_AUTH_ACCESS_TOKEN=$CLOUDSDK_AUTH_ACCESS_TOKEN" 49 | fi 50 | 51 | USER=$(docker inspect --format='{{"{{.Config.User}}"}}' {{ service.instance_name }}_toolbox) 52 | if [ -z "$USER" ]; then 53 | USER="root" 54 | fi 55 | 56 | # Start & exec into docker container 57 | docker run --user "$USER" --entrypoint sh \ 58 | -e "$docker_auth_var" \ 59 | {%- for variable in toolbox.environment_items -%} 60 | -e {{ variable.name }}={{ shlex.quote(variable.value) }} \ 61 | {%- endfor -%} 62 | --network ${CNC_TOOLBOX_NETWORK:-host} \ 63 | {%- if command %} 64 | --rm {{ service.instance_name }}_toolbox -c '{{ command }}'; command_result=$? 65 | {%- else %} 66 | -it --rm {{ service.instance_name }}_toolbox 67 | {%- endif %} 68 | 69 | {%- if command %} 70 | exit $command_result 71 | {%- endif %} -------------------------------------------------------------------------------- /src/cnc/logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | from inspect import currentframe 3 | from rich import print 4 | 5 | 6 | def get_logger(name="root"): 7 | return Logger(name) 8 | 9 | 10 | class Logger: 11 | def __init__(self, name): 12 | self.name = name 13 | 14 | def debug(self, msg): 15 | debug = bool(os.environ.get("CNC_DEBUG", True)) 16 | if debug: 17 | print( 18 | f"[green]DEBUG ({self.name}:{currentframe().f_back.f_lineno})" 19 | f"[/ green] {msg}" 20 | ) 21 | 22 | def info(self, msg): 23 | print( 24 | f"[blue]INFO ({self.name}:{currentframe().f_back.f_lineno})" 25 | f"[/ blue] {msg}" 26 | ) 27 | 28 | def warning(self, msg): 29 | print( 30 | f"[yellow]WARNING ({self.name}:{currentframe().f_back.f_lineno})" 31 | f"[/ yellow] {msg}" 32 | ) 33 | 34 | def error(self, msg): 35 | print( 36 | f"[red]ERROR ({self.name}:{currentframe().f_back.f_lineno})" 37 | f"[/ red] {msg}" 38 | ) 39 | -------------------------------------------------------------------------------- /src/cnc/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_model import BaseModel, IgnoredType 2 | from .environment_variable import EnvironmentVariable 3 | from .config import AppConfig, InternalSettings 4 | from .environment import Environment 5 | from .environment_collection import EnvironmentCollection 6 | from .application import Application 7 | from .builder import BuildStageManager 8 | from .deployer import DeployStageManager 9 | from .provisioner import ProvisionStageManager 10 | from .toolbox import ToolboxManager 11 | from .resource_use_existing import ResourceUseExistingSettings 12 | -------------------------------------------------------------------------------- /src/cnc/models/base_model.py: -------------------------------------------------------------------------------- 1 | from pydantic import ( 2 | BaseModel as PydanticBaseModel, 3 | model_validator, 4 | ValidationInfo, 5 | ) 6 | 7 | from cnc.logger import get_logger 8 | 9 | log = get_logger(__name__) 10 | 11 | 12 | class IgnoredType: 13 | pass 14 | 15 | 16 | class BaseModel(PydanticBaseModel): 17 | @model_validator(mode="before") 18 | @classmethod 19 | def ensure_provider(cls, data: dict, info: ValidationInfo): 20 | if info.context: 21 | data["provider"] = info.context.get("provider") 22 | 23 | return data 24 | 25 | @classmethod 26 | def convert_dict_keys_to_names(cls, value: dict) -> list[dict]: 27 | with_names = [] 28 | 29 | for k, v in value.items(): 30 | if not isinstance(v, dict): 31 | raise ValueError(f"{k} is not a dict, bad data format") 32 | 33 | v.update({"name": k}) 34 | with_names.append(v) 35 | 36 | return with_names 37 | 38 | class Config: 39 | ignored_types = (IgnoredType,) 40 | arbitrary_types_allowed = True 41 | -------------------------------------------------------------------------------- /src/cnc/models/config/__init__.py: -------------------------------------------------------------------------------- 1 | from .config import AppConfig, InternalSettings 2 | -------------------------------------------------------------------------------- /src/cnc/models/config/deploy_resource_limits.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Union 2 | from pydantic import Field 3 | 4 | from cnc.models.base_model import BaseModel 5 | from cnc.logger import get_logger 6 | 7 | log = get_logger(__name__) 8 | 9 | 10 | class DeployResourceLimits(BaseModel): 11 | raw_memory: Optional[str] = Field(default="512Mi", alias="memory") 12 | raw_cpu: Optional[Union[int, float]] = Field(default=1, alias="cpus") 13 | -------------------------------------------------------------------------------- /src/cnc/models/config/utils.py: -------------------------------------------------------------------------------- 1 | def validate_command_list(raw_cmd): 2 | if isinstance(raw_cmd, str): 3 | return raw_cmd.split() 4 | return raw_cmd 5 | -------------------------------------------------------------------------------- /src/cnc/models/custom_header.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, List 2 | 3 | from .base_model import BaseModel 4 | 5 | 6 | class CustomHeader(BaseModel): 7 | name: str 8 | value: str 9 | type: Optional[str] = "response" 10 | 11 | 12 | class CustomHeaders(BaseModel): 13 | headers: List[CustomHeader] = [] 14 | provider: str 15 | -------------------------------------------------------------------------------- /src/cnc/models/providers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/src/cnc/models/providers/__init__.py -------------------------------------------------------------------------------- /src/cnc/models/providers/amazon/__init__.py: -------------------------------------------------------------------------------- 1 | from .service_account import AmazonAppServiceAccount 2 | from .custom_headers import AWSCustomHeaders 3 | from .database_resource_settings import AWSDatabaseResourceSettings 4 | from .cache_resource_settings import AWSCacheResourceSettings 5 | from .deploy_resource_limits import AWSDeployResourceLimits 6 | -------------------------------------------------------------------------------- /src/cnc/models/providers/amazon/cache_resource_settings.py: -------------------------------------------------------------------------------- 1 | from typing import Literal 2 | 3 | from cnc.models.config.resource import CacheResourceSettings 4 | from cnc.logger import get_logger 5 | 6 | log = get_logger(__name__) 7 | 8 | 9 | class AWSCacheResourceSettings(CacheResourceSettings): 10 | provider: Literal["aws"] 11 | 12 | @property 13 | def toolbox_ssh_port_mapping(self): 14 | return ( 15 | "{" 16 | f'"host":["{self.redis_ip}"],"portNumber":["{self.redis_port}"], ' 17 | f'"localPortNumber":["{self.local_port}"]' 18 | "}" 19 | ) 20 | -------------------------------------------------------------------------------- /src/cnc/models/providers/amazon/service_account.py: -------------------------------------------------------------------------------- 1 | from cnc.logger import get_logger 2 | 3 | log = get_logger(__name__) 4 | 5 | 6 | class AmazonAppServiceAccount: 7 | def __init__(self, collection): 8 | self.collection = collection 9 | -------------------------------------------------------------------------------- /src/cnc/models/providers/google/__init__.py: -------------------------------------------------------------------------------- 1 | from .database_resource_settings import GCPDatabaseResourceSettings 2 | from .cache_resource_settings import GCPCacheResourceSettings 3 | from .deploy_resource_limits import GCPDeployResourceLimits 4 | -------------------------------------------------------------------------------- /src/cnc/models/providers/google/cache_resource_settings.py: -------------------------------------------------------------------------------- 1 | from typing import Literal 2 | from cnc.models.config.resource import CacheResourceSettings 3 | from cnc.logger import get_logger 4 | 5 | log = get_logger(__name__) 6 | 7 | 8 | class GCPCacheResourceSettings(CacheResourceSettings): 9 | provider: Literal["gcp"] 10 | 11 | # ------------------------------ 12 | # Properties 13 | # ------------------------------ 14 | 15 | @property 16 | def provider_version(self): 17 | _version = float(self.version) 18 | if _version == 7.2: 19 | return "REDIS_7_2" 20 | 21 | major_version = int(_version) 22 | if major_version == 6: 23 | return "REDIS_6_X" 24 | 25 | return "REDIS_{{ service.settings.version }}_0" 26 | 27 | @property 28 | def toolbox_ssh_port_mapping(self): 29 | return f"{self.local_port}:{self.redis_ip}:{self.redis_port}" 30 | -------------------------------------------------------------------------------- /src/cnc/models/providers/google/database_resource_settings.py: -------------------------------------------------------------------------------- 1 | from typing import Literal 2 | from cnc.models.config.resource import DatabaseResourceSettings 3 | from cnc.logger import get_logger 4 | 5 | log = get_logger(__name__) 6 | 7 | 8 | class GCPDatabaseResourceSettings(DatabaseResourceSettings): 9 | provider: Literal["gcp"] 10 | 11 | # ------------------------------ 12 | # Properties 13 | # ------------------------------ 14 | 15 | @property 16 | def password_secret_id(self): 17 | return f"{self.service.instance_name}_password" 18 | 19 | @property 20 | def url_secret_id(self): 21 | return f"{self.service.instance_name}_url" 22 | 23 | @property 24 | def host_output_tf_value_string(self): 25 | tf_resource_type = "google_sql_database_instance." 26 | host_attr = ".private_ip_address" 27 | 28 | if "lite" in self.application.flavor: 29 | host_attr = ".public_ip_address" 30 | 31 | if self.use_existing: 32 | tf_resource_type = f"data.{tf_resource_type}" 33 | 34 | return "".join( 35 | ["${", tf_resource_type, self.service.instance_name, host_attr, "}"] 36 | ) 37 | 38 | @property 39 | def password_from_db_secret(self): 40 | try: 41 | return self.collection.get_secret_value(self.password_secret_id) 42 | except Exception as e: 43 | log.debug(f"No existing db secret found for {self.service}: {e}") 44 | return None 45 | 46 | @property 47 | def toolbox_managed_environment_variables(self): 48 | if "lite" in self.application.flavor: 49 | _env = self.managed_environment_variables 50 | 51 | db_password = self.database_password 52 | db_url = self.database_url(db_password=db_password) 53 | _env.update( 54 | { 55 | f"{self.env_var_base}_DATABASE_URL": db_url, 56 | "DATABASE_URL": db_url, 57 | f"{self.env_var_base}_DB_PASSWORD": db_password, 58 | "DB_PASSWORD": db_password, 59 | } 60 | ) 61 | return _env 62 | 63 | return super().toolbox_managed_environment_variables 64 | 65 | @property 66 | def toolbox_ssh_port_mapping(self): 67 | return f"{self.local_port}:{self.database_endpoint}:{self.remote_port}" 68 | -------------------------------------------------------------------------------- /src/cnc/models/providers/google/service_account.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/src/cnc/models/providers/google/service_account.py -------------------------------------------------------------------------------- /src/cnc/models/resource_use_existing.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, List 2 | 3 | from .base_model import BaseModel, IgnoredType 4 | 5 | 6 | # TODO: Should we use a discriminator here so we 7 | # can have better validations etc? 8 | class ResourceUseExistingSettings(BaseModel): 9 | name: str 10 | instance_name: str 11 | secret_id: Optional[str] = None 12 | manage_databases: Optional[bool] = True 13 | username: Optional[str] = None 14 | db_name: Optional[str] = None 15 | cluster_mode: Optional[bool] = False 16 | public_subnet_cidrs: Optional[List[str]] = None 17 | private_subnet_cidrs: Optional[List[str]] = None 18 | public_subnet_ids: Optional[List[str]] = None 19 | private_subnet_ids: Optional[List[str]] = None 20 | 21 | # ------------------------------ 22 | # Parent relationships 23 | # ------------------------------ 24 | 25 | settings: Optional[IgnoredType] = IgnoredType() 26 | -------------------------------------------------------------------------------- /src/cnc/models/stage.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from .base_model import BaseModel 4 | 5 | 6 | class Stage(BaseModel): 7 | name: str 8 | provider_token: Optional[str] = "" 9 | callback_url: Optional[str] = "" 10 | 11 | def __getattribute__(self, item): 12 | # Override to catch "is_" pattern 13 | if item.startswith("is_"): 14 | name_to_check = item[3:] 15 | 16 | if name_to_check not in ["build", "deploy"]: 17 | # preserve default attributes of BaseModel if not a valid cnc stage name 18 | return super().__getattribute__(item) 19 | 20 | # Compare with the instance's name property 21 | if name_to_check == super().__getattribute__("name"): 22 | return True 23 | return False 24 | else: 25 | return super().__getattribute__(item) 26 | -------------------------------------------------------------------------------- /src/cnc/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coherenceplatform/cnc/11243b852042c97db7a16edd8de2da2e95f1e229/src/cnc/tests/__init__.py -------------------------------------------------------------------------------- /src/cnc/tests/base_test_class.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import shutil 4 | import uuid 5 | 6 | 7 | class CNCBaseTestCase(unittest.TestCase): 8 | fixture_name = "" 9 | env_data_filepath = "environments.yml" 10 | 11 | def setUp(self): 12 | if not self.fixture_name: 13 | raise ValueError("Must provide fixture_name") 14 | 15 | self.working_dir = f"/tmp/{str(uuid.uuid4())}" 16 | 17 | shared_template_dir = f"{os.path.dirname(__file__)}/fixtures/shared" 18 | fixtures_path = f"{os.path.dirname(__file__)}/fixtures/{self.fixture_name}" 19 | 20 | shutil.copytree( 21 | shared_template_dir, 22 | self.working_dir, 23 | dirs_exist_ok=True, 24 | ) 25 | 26 | shutil.copytree( 27 | fixtures_path, 28 | self.working_dir, 29 | dirs_exist_ok=True, 30 | ) 31 | 32 | # print(f"\n{self.working_dir}") 33 | # subprocess.run(["ls", "-al", self.working_dir]) 34 | os.chdir(self.working_dir) 35 | # subprocess.run(["cat", "cnc.yml"]) 36 | # print("\n") 37 | 38 | def tearDown(self): 39 | shutil.rmtree( 40 | self.working_dir, 41 | ignore_errors=True, 42 | ) 43 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-serverless-1-service-1-db-1-dynamo-1/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | url_path: /app 6 | build: 7 | context: . 8 | 9 | db1: 10 | x-cnc: 11 | type: database 12 | version: 15 13 | image: postgres 14 | 15 | lambda1: 16 | x-cnc: 17 | type: serverless 18 | url_path: /lambda1 19 | build: 20 | context: . 21 | 22 | dynamodb1: 23 | x-cnc: 24 | type: dynamodb 25 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-serverless-1-service-1-db-1-dynamo-1/environments_backend_1_serverless_1_service_1_db_1_dynamo.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | data: 12 | infrastructure_outputs: 13 | hosted_zone_ns_records: 14 | value: ['record1', 'record2'] 15 | 16 | environments: 17 | - name: main 18 | environment_variables: 19 | - name: foo 20 | value: bar 21 | - name: type 22 | value: "serverless" -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-serverless-1-service-1-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | url_path: /app 6 | build: 7 | context: . 8 | 9 | db1: 10 | x-cnc: 11 | type: database 12 | version: 15 13 | image: postgres 14 | 15 | lambda1: 16 | x-cnc: 17 | type: serverless 18 | url_path: /lambda1 19 | build: 20 | context: . -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-serverless-1-service-1-db/environments_backend_1_serverless_1_service_1_db.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | data: 12 | infrastructure_outputs: 13 | hosted_zone_ns_records: 14 | value: ['record1', 'record2'] 15 | 16 | environments: 17 | - name: main 18 | environment_variables: 19 | - name: foo 20 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-serverless-1-service-2-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | url_path: /app 6 | build: 7 | context: . 8 | 9 | db1: 10 | x-cnc: 11 | type: database 12 | version: 15 13 | image: postgres 14 | 15 | db2: 16 | x-cnc: 17 | type: database 18 | version: 15 19 | image: postgres 20 | 21 | lambda1: 22 | x-cnc: 23 | type: serverless 24 | url_path: /lambda1 25 | build: 26 | context: . -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-serverless-1-service-2-db/environments_backend_1_serverless_1_service_2_db.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | data: 12 | infrastructure_outputs: 13 | hosted_zone_ns_records: 14 | value: ['record1', 'record2'] 15 | 16 | environments: 17 | - name: main 18 | environment_variables: 19 | - name: foo 20 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-serverless-2-service-2-db-2-dynamo-2/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | url_path: /app 6 | build: 7 | context: . 8 | 9 | db1: 10 | x-cnc: 11 | type: database 12 | version: 15 13 | image: postgres 14 | 15 | db2: 16 | x-cnc: 17 | type: database 18 | version: 15 19 | image: postgres 20 | 21 | lambda1: 22 | x-cnc: 23 | type: serverless 24 | url_path: /lambda1 25 | build: 26 | context: . 27 | 28 | lambda2: 29 | x-cnc: 30 | type: serverless 31 | url_path: /lambda2 32 | build: 33 | context: . 34 | 35 | dynamodb1: 36 | x-cnc: 37 | type: dynamodb 38 | 39 | dynamodb2: 40 | x-cnc: 41 | type: dynamodb 42 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-serverless-2-service-2-db-2-dynamo-2/environments_backend_1_serverless_2_service_2_db_2_dynamo.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | data: 12 | infrastructure_outputs: 13 | hosted_zone_ns_records: 14 | value: ['record1', 'record2'] 15 | 16 | environments: 17 | - name: main 18 | environment_variables: 19 | - name: foo 20 | value: bar 21 | - name: type 22 | value: "serverless" -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-db-both-use-existing/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . 7 | 8 | db1: 9 | x-cnc: 10 | type: database 11 | version: 15 12 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-db-both-use-existing/environments.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | existing_resources: 12 | db1: 13 | instance_name: foobar 14 | secret_id: foo123-1 15 | environments: 16 | - name: main 17 | environment_variables: 18 | - name: foo 19 | value: bar 20 | existing_resources: 21 | db1: 22 | instance_name: bazbar 23 | secret_id: baz123-1 -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-db-use-existing/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . 7 | 8 | db1: 9 | x-cnc: 10 | type: database 11 | version: 15 12 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-db-use-existing/environments.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | existing_resources: 12 | db1: 13 | instance_name: foobar 14 | secret_id: foo123-1 15 | environments: 16 | - name: main 17 | environment_variables: 18 | - name: foo 19 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-db-variables-aliases/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . 7 | 8 | db1: 9 | x-cnc: 10 | type: database 11 | version: 15 12 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-db-variables-aliases/environments.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: foo-standard 15 | value: bar 16 | - name: foo-secret 17 | secret_id: bar123 18 | - name: foo-output 19 | output_name: bar123-a 20 | - name: foo-standard-alias 21 | alias: foo-standard 22 | - name: foo-secret-alias 23 | alias: foo-secret 24 | - name: foo-output-alias 25 | alias: foo-output -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . 7 | 8 | db1: 9 | x-cnc: 10 | type: database 11 | version: 15 12 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-db/environments_aws_ecs_existing_vpc_cidrs.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | existing_resources: 12 | vpc: 13 | instance_name: vpc-0aa28c08518eda64e 14 | public_subnet_cidrs: 15 | - 10.0.0.0/24 16 | - 10.0.1.0/24 17 | private_subnet_cidrs: 18 | - 10.0.2.0/24 19 | - 10.0.3.0/24 20 | data: 21 | infrastructure_outputs: 22 | hosted_zone_ns_records: 23 | value: ['record1', 'record2'] 24 | 25 | environments: 26 | - name: main 27 | environment_variables: 28 | - name: foo 29 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-db/environments_aws_ecs_existing_vpc_subnets.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | existing_resources: 12 | vpc: 13 | instance_name: vpc-0aa28c08518eda64e 14 | public_subnet_ids: 15 | - "subnet-04e772928102604dc" 16 | - "subnet-02e772928102604dc" 17 | private_subnet_ids: 18 | - "subnet-03e772928102604dc" 19 | - "subnet-05e772928102604dc" 20 | data: 21 | infrastructure_outputs: 22 | hosted_zone_ns_records: 23 | value: ['record1', 'record2'] 24 | 25 | environments: 26 | - name: main 27 | environment_variables: 28 | - name: foo 29 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-1-worker-1-task/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | workers: 6 | - name: worker-1 7 | command: "python -m cnc.worker" 8 | system: 9 | cpus: 1 10 | 11 | scheduled_tasks: 12 | - name: task-1 13 | command: "python -m cnc.task" 14 | schedule: "0 0 * * *" 15 | system: 16 | memory: 1G 17 | build: 18 | context: . 19 | deploy: 20 | resources: 21 | limits: 22 | cpus: 2 23 | memory: 4G 24 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-2-db-2-envs/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . 7 | 8 | db1: 9 | x-cnc: 10 | type: database 11 | version: 15 12 | image: postgres 13 | 14 | db2: 15 | x-cnc: 16 | type: database 17 | version: 15 18 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-2-db-2-envs/environments.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: foo 15 | value: bar 16 | - name: staging 17 | environment_variables: 18 | - name: foo2 19 | value: baz -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-2-db-2-envs/environments_aws_ecs.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | data: 12 | infrastructure_outputs: 13 | hosted_zone_ns_records: 14 | value: ['record1', 'record2'] 15 | environments: 16 | - name: main 17 | environment_variables: 18 | - name: foo 19 | value: bar 20 | - name: staging 21 | environment_variables: 22 | - name: foo2 23 | value: baz 24 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-2-db-2-envs/environments_gcp_run_region_settings.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | region: us-west2 12 | environments: 13 | - name: main 14 | environment_variables: 15 | - name: foo 16 | value: bar 17 | - name: staging 18 | environment_variables: 19 | - name: foo2 20 | value: baz -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-2-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . 7 | 8 | db1: 9 | x-cnc: 10 | type: database 11 | version: 15 12 | image: postgres 13 | 14 | db2: 15 | x-cnc: 16 | type: database 17 | version: 15 18 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-custom/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-custom/custom/deploy/main.sh.j2: -------------------------------------------------------------------------------- 1 | echo "custom_template" -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-custom/environments.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | 7 | template_config: 8 | template_directory: custom 9 | 10 | collections: 11 | - name: preview 12 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 13 | account_id: "foo-bar-123" 14 | environments: 15 | - name: main 16 | environment_variables: 17 | - name: foo 18 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-existing/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | image: cnc-app:latest 6 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service-invalid/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . 7 | badapp: 8 | x-cnc: 9 | type: nobackend 10 | build: 11 | context: . -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-1-service/environments_json_var.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | allowed_hosts: ["my-extra-allowed-host.cnctest.com"] 14 | environment_variables: 15 | - name: foo 16 | value: "{\"type\":\"backend\"}" 17 | - name: bar 18 | value: "non-json stuff" 19 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-2-serverless-2-service-2-db-2-dynamo-2/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | url_path: /app 6 | build: 7 | context: . 8 | 9 | db1: 10 | x-cnc: 11 | type: database 12 | version: 15 13 | image: postgres 14 | 15 | lambda1: 16 | x-cnc: 17 | type: serverless 18 | url_path: /lambda1 19 | build: 20 | context: . 21 | 22 | dynamodb1: 23 | x-cnc: 24 | type: dynamodb 25 | 26 | app2: 27 | x-cnc: 28 | type: backend 29 | url_path: /app2 30 | build: 31 | context: . 32 | 33 | db2: 34 | x-cnc: 35 | type: database 36 | version: 15 37 | image: postgres 38 | 39 | lambda2: 40 | x-cnc: 41 | type: serverless 42 | url_path: /lambda2 43 | build: 44 | context: . 45 | 46 | dynamodb2: 47 | x-cnc: 48 | type: dynamodb 49 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-2-serverless-2-service-2-db-2-dynamo-2/environments_backend_2_serverless_2_service_2_db_2_dynamo.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | data: 12 | infrastructure_outputs: 13 | hosted_zone_ns_records: 14 | value: ['record1', 'record2'] 15 | 16 | environments: 17 | - name: main 18 | environment_variables: 19 | - name: foo 20 | value: bar 21 | - name: type 22 | value: "serverless" -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-2-serverless-2-service-2-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | url_path: /app 6 | build: 7 | context: . 8 | 9 | db1: 10 | x-cnc: 11 | type: database 12 | version: 15 13 | image: postgres 14 | 15 | lambda1: 16 | x-cnc: 17 | type: serverless 18 | url_path: /lambda1 19 | build: 20 | context: . 21 | 22 | api: 23 | x-cnc: 24 | type: backend 25 | url_path: /api 26 | build: 27 | context: . 28 | 29 | db2: 30 | x-cnc: 31 | type: database 32 | version: 15 33 | image: postgres 34 | 35 | lambda2: 36 | x-cnc: 37 | type: serverless 38 | url_path: /lambda2 39 | build: 40 | context: . -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-2-serverless-2-service-2-db/environments_backend_2_serverless_2_service_2_db.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | data: 12 | infrastructure_outputs: 13 | hosted_zone_ns_records: 14 | value: ['record1', 'record2'] 15 | 16 | environments: 17 | - name: main 18 | environment_variables: 19 | - name: foo 20 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-2-service-1-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: ./app 7 | 8 | api: 9 | x-cnc: 10 | type: backend 11 | url_path: /api 12 | build: 13 | context: ./api 14 | 15 | db1: 16 | x-cnc: 17 | type: database 18 | version: 15 19 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/backend-2-service-2-db/cnc.yml: -------------------------------------------------------------------------------- 1 | x-cnc: 2 | resources: 3 | - name: db2 4 | type: database 5 | version: 15 6 | engine: postgres 7 | 8 | services: 9 | app: 10 | x-cnc: 11 | type: backend 12 | build: 13 | context: ./app 14 | 15 | api: 16 | x-cnc: 17 | type: backend 18 | url_path: /api 19 | build: 20 | context: ./api 21 | 22 | db1: 23 | x-cnc: 24 | type: database 25 | version: 15 26 | image: postgres:15 -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-1-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: serverless 5 | build: 6 | context: . 7 | 8 | db1: 9 | x-cnc: 10 | type: database 11 | version: 15 12 | use_db_proxy: False 13 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-1-db/environments_serverless_1_service_1_db.yml: -------------------------------------------------------------------------------- 1 | name: my-serverless-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: lambda-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-serverless-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: type 15 | value: "serverless" 16 | - name: bar 17 | value: "non-json stuff" 18 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-1-dynamodb-1-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: serverless 5 | build: 6 | context: . 7 | 8 | dynamodb1: 9 | x-cnc: 10 | type: dynamodb 11 | 12 | db1: 13 | x-cnc: 14 | type: database 15 | version: 15 16 | use_db_proxy: False 17 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-1-dynamodb-1-db/environments_serverless_1_service_1_dynamodb_1_db.yml: -------------------------------------------------------------------------------- 1 | name: my-serverless-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: lambda-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-serverless-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: type 15 | value: "serverless" 16 | - name: bar 17 | value: "non-json stuff" 18 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-1-dynamodb/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: serverless 5 | build: 6 | context: . 7 | 8 | dynamodb1: 9 | x-cnc: 10 | type: dynamodb 11 | 12 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-1-dynamodb/environments_serverless_1_service_1_dynamodb.yml: -------------------------------------------------------------------------------- 1 | name: my-serverless-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: lambda-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-serverless-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: type 15 | value: "serverless" 16 | - name: bar 17 | value: "non-json stuff" 18 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-2-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: serverless 5 | build: 6 | context: . 7 | 8 | db1: 9 | x-cnc: 10 | type: database 11 | version: 15 12 | use_db_proxy: False 13 | image: postgres 14 | 15 | db2: 16 | x-cnc: 17 | type: database 18 | version: 15 19 | use_db_proxy: False 20 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-2-db/environments_serverless_1_service_2_db.yml: -------------------------------------------------------------------------------- 1 | name: my-serverless-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: lambda-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-serverless-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: type 15 | value: "serverless" 16 | - name: bar 17 | value: "non-json stuff" 18 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-2-dynamodb-2-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: serverless 5 | build: 6 | context: . 7 | 8 | dynamodb1: 9 | x-cnc: 10 | type: dynamodb 11 | 12 | dynamodb2: 13 | x-cnc: 14 | type: dynamodb 15 | 16 | db1: 17 | x-cnc: 18 | type: database 19 | version: 15 20 | use_db_proxy: False 21 | image: postgres 22 | 23 | db2: 24 | x-cnc: 25 | type: database 26 | version: 15 27 | use_db_proxy: False 28 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-2-dynamodb-2-db/environments_serverless_1_service_2_dynamodb_2_db.yml: -------------------------------------------------------------------------------- 1 | name: my-serverless-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: lambda-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-serverless-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: type 15 | value: "serverless" 16 | - name: bar 17 | value: "non-json stuff" 18 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-2-dynamodb/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: serverless 5 | build: 6 | context: . 7 | 8 | 9 | dynamodb1: 10 | x-cnc: 11 | type: dynamodb 12 | 13 | 14 | dynamodb2: 15 | x-cnc: 16 | type: dynamodb 17 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service-2-dynamodb/environments_serverless_1_service_2_dynamodb.yml: -------------------------------------------------------------------------------- 1 | name: my-serverless-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: lambda-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-serverless-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: type 15 | value: "serverless" 16 | - name: bar 17 | value: "non-json stuff" 18 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: serverless 5 | build: 6 | context: . -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-1-service/environments_serverless_1_service.yml: -------------------------------------------------------------------------------- 1 | name: my-serverless-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: lambda-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-serverless-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: type 15 | value: "serverless" 16 | - name: bar 17 | value: "non-json stuff" 18 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-2-service-2-dynamodb-2-db/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: serverless 5 | build: 6 | context: . 7 | 8 | app2: 9 | x-cnc: 10 | type: serverless 11 | build: 12 | context: . 13 | 14 | dynamodb1: 15 | x-cnc: 16 | type: dynamodb 17 | 18 | dynamodb2: 19 | x-cnc: 20 | type: dynamodb 21 | 22 | db1: 23 | x-cnc: 24 | type: database 25 | use_db_proxy: False 26 | version: 15 27 | image: postgres 28 | 29 | db2: 30 | x-cnc: 31 | type: database 32 | version: 15 33 | use_db_proxy: False 34 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-2-service-2-dynamodb-2-db/environments_serverless_2_service_2_dynamodb_2_db.yml: -------------------------------------------------------------------------------- 1 | name: my-serverless-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: lambda-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-serverless-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: type 15 | value: "serverless" 16 | - name: bar 17 | value: "non-json stuff" 18 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-2-service-2-dynamodb/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: serverless 5 | build: 6 | context: . 7 | 8 | app2: 9 | x-cnc: 10 | type: serverless 11 | build: 12 | context: . 13 | 14 | dynamodb1: 15 | x-cnc: 16 | type: dynamodb 17 | 18 | 19 | dynamodb2: 20 | x-cnc: 21 | type: dynamodb 22 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/serverless-2-service-2-dynamodb/environments_serverless_2_service_2_dynamodb.yml: -------------------------------------------------------------------------------- 1 | name: my-serverless-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: lambda-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-serverless-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: type 15 | value: "serverless" 16 | - name: bar 17 | value: "non-json stuff" 18 | -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/shared/cnc.yml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | x-cnc: 4 | type: backend 5 | build: 6 | context: . 7 | 8 | db1: 9 | x-cnc: 10 | type: database 11 | version: 15 12 | image: postgres -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/shared/environments-custom-provision.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | template_config: 7 | template_directory: custom 8 | 9 | collections: 10 | - name: preview 11 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 12 | account_id: "foo-bar-123" 13 | environments: 14 | - name: main 15 | environment_variables: 16 | - name: foo 17 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/shared/environments.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: foo 15 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/shared/environments_aws_ecs.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | data: 12 | infrastructure_outputs: 13 | hosted_zone_ns_records: 14 | value: ['record1', 'record2'] 15 | 16 | environments: 17 | - name: main 18 | environment_variables: 19 | - name: foo 20 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/shared/environments_aws_ecs_existing_vpc.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: aws 3 | region: us-east-1 4 | flavor: ecs 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | existing_resources: 12 | vpc: 13 | instance_name: vpc-0aa28c08518eda64e 14 | data: 15 | infrastructure_outputs: 16 | hosted_zone_ns_records: 17 | value: ['record1', 'record2'] 18 | 19 | environments: 20 | - name: main 21 | environment_variables: 22 | - name: foo 23 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/shared/environments_gcp_run_lite.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run-lite 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: foo 15 | value: bar -------------------------------------------------------------------------------- /src/cnc/tests/fixtures/shared/environments_weird_character_vars.yml: -------------------------------------------------------------------------------- 1 | name: my-backend-test-app 2 | provider: gcp 3 | region: us-east1 4 | flavor: run 5 | version: 1 6 | 7 | collections: 8 | - name: preview 9 | base_domain: my-backend-test-app.testnewsite.coherencesites.com 10 | account_id: "foo-bar-123" 11 | environments: 12 | - name: main 13 | environment_variables: 14 | - name: FOO 15 | value: bar 16 | - name: FOO_JSON 17 | value: '{"foo": "bar"}' 18 | - name: FOO_SPECIALS 19 | value: > 20 | foo!"'`@#$%^&*()_+bar\ 21 | - name: COMPLEX_JSON 22 | value: '{"key1": ["value1", "value2"], "key2": {"nested": true}}' 23 | - name: MULTILINE 24 | value: > 25 | line1 26 | line2 indented 27 | - name: COMPLEX_PASSWORD 28 | value: 'P@ssw0rd!#$%^&*()_+{}[]|\/?,.<>~`' -------------------------------------------------------------------------------- /src/cnc/tests/test_environment_collection.py: -------------------------------------------------------------------------------- 1 | from .base_test_class import CNCBaseTestCase 2 | from cnc.models import Application 3 | 4 | from cnc.logger import get_logger 5 | 6 | log = get_logger(__name__) 7 | 8 | 9 | class EnvironmentCollectionBaseTestCase(CNCBaseTestCase): 10 | fixture_name = "backend-1-service-1-db" 11 | env_data_filepath = "environments.yml" 12 | 13 | def setUp(self): 14 | super().setUp() 15 | app = Application.from_environments_yml(self.env_data_filepath) 16 | self.collection = app.collections[0] 17 | 18 | 19 | class EnvironmentCollectionTestCase(EnvironmentCollectionBaseTestCase): 20 | def test_collection_env_by_name(self): 21 | self.assertIsNone(self.collection.environment_by_name("foo")) 22 | self.assertIsNotNone(self.collection.environment_by_name("main")) 23 | 24 | 25 | class EnvironmentCollectionExistingDBTestCase(EnvironmentCollectionBaseTestCase): 26 | fixture_name = "backend-1-service-1-db-use-existing" 27 | 28 | def test_collection_use_existing_db(self): 29 | self.assertEqual( 30 | self.collection.environments[0].existing_resources[0].name, "db1" 31 | ) 32 | self.assertEqual( 33 | self.collection.environments[0].existing_resources[0].instance_name, 34 | "foobar", 35 | ) 36 | 37 | 38 | class EnvironmentCollectionTwoServicesandDBsTwoEnvsTest( 39 | EnvironmentCollectionBaseTestCase 40 | ): 41 | fixture_name = "backend-1-service-2-db-2-envs" 42 | 43 | def test_environment_order(self): 44 | self.assertEqual(len(self.collection.environments), 2) 45 | self.assertEqual(self.collection.environments[0].name, "main") 46 | self.assertEqual(self.collection.environments[1].name, "staging") 47 | 48 | def test_environment_services(self): 49 | self.assertEqual(len(self.collection.all_services), 6) 50 | self.assertEqual(len(self.collection.all_services_for_type("backend")), 2) 51 | self.assertEqual(len(self.collection.all_services_for_type("database")), 4) 52 | self.assertEqual(len(self.collection.all_services_for_type("frontend")), 0) 53 | self.assertEqual(len(self.collection.all_services_for_type("cache")), 0) 54 | 55 | 56 | class EnvironmentCollectionRegionSettings(EnvironmentCollectionBaseTestCase): 57 | fixture_name = "backend-1-service-2-db-2-envs" 58 | env_data_filepath = "environments_gcp_run_region_settings.yml" 59 | 60 | def test_region_settings(self): 61 | self.assertEqual(self.collection.region, self.collection.collection_region) 62 | self.assertEqual(self.collection.region, "us-west2") 63 | self.assertEqual(self.collection.application.region, "us-east1") 64 | -------------------------------------------------------------------------------- /src/cnc/tests/test_environment_variables.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | from .base_test_class import CNCBaseTestCase 4 | from cnc.models import Application 5 | from cnc.models.providers.google.environment_collection import GCPEnvironmentCollection 6 | 7 | from cnc.logger import get_logger 8 | 9 | log = get_logger(__name__) 10 | 11 | 12 | class EnvironmentVariablesTestCase(CNCBaseTestCase): 13 | fixture_name = "backend-1-service-1-db-variables-aliases" 14 | 15 | def setUp(self): 16 | super().setUp() 17 | app = Application.from_environments_yml("environments.yml") 18 | self.collection = app.collections[0] 19 | self.environment = self.collection.environments[0] 20 | self.assertEqual(self.environment.name, "main") 21 | 22 | def test_variables(self): 23 | self.assertEqual(len(self.environment.environment_variables), 6) 24 | 25 | _std = self.environment.environment_variables[0] 26 | self.assertEqual(_std.name, "foo-standard") 27 | self.assertEqual(_std.value, "bar") 28 | self.assertEqual(_std.secret_id, None) 29 | self.assertEqual(_std.output_name, None) 30 | self.assertEqual(_std.alias, None) 31 | 32 | _secret = self.environment.environment_variables[1] 33 | with patch.object( 34 | GCPEnvironmentCollection, "get_secret_value", return_value="secretvalue" 35 | ) as secret_mock: 36 | self.assertEqual(_secret.value, "secretvalue") 37 | secret_mock.assert_called_once 38 | 39 | self.assertEqual(_secret.secret_id, "bar123") 40 | self.assertEqual(_secret.output_name, None) 41 | self.assertEqual(_secret.alias, None) 42 | 43 | _std_alias = self.environment.environment_variables[3] 44 | self.assertEqual(_std_alias.name, "foo-standard-alias") 45 | self.assertEqual(_std_alias.secret_id, None) 46 | self.assertEqual(_std_alias.output_name, None) 47 | self.assertEqual(_std_alias.alias, "foo-standard") 48 | self.assertEqual(_std.value, "bar") 49 | 50 | def test_items_property(self): 51 | item_names = [item.name for item in self.environment.environment_items] 52 | 53 | # variable 54 | self.assertIn("foo-standard", item_names) 55 | # managed var 56 | self.assertIn("DB_PASSWORD", item_names) 57 | # managed secret 58 | self.assertIn("DB1_ENDPOINT", item_names) 59 | 60 | # check that there are no dupe names 61 | self.assertEqual(len(item_names), len(set(item_names))) 62 | -------------------------------------------------------------------------------- /src/cnc/tests/test_environments.py: -------------------------------------------------------------------------------- 1 | from .base_test_class import CNCBaseTestCase 2 | from cnc.models import Application 3 | 4 | from cnc.logger import get_logger 5 | 6 | log = get_logger(__name__) 7 | 8 | 9 | class EnvironmentBaseTestCase(CNCBaseTestCase): 10 | fixture_name = "backend-1-service-1-db" 11 | 12 | def setUp(self): 13 | super().setUp() 14 | app = Application.from_environments_yml("environments.yml") 15 | self.collection = app.collections[0] 16 | self.environment = self.collection.environments[0] 17 | self.assertEqual(self.environment.name, "main") 18 | 19 | 20 | class EnvironmentUseExistingDBTestCase(EnvironmentBaseTestCase): 21 | fixture_name = "backend-1-service-1-db-use-existing" 22 | 23 | def test_environment_use_existing_db(self): 24 | self.assertEqual(len(self.collection.environments), 1) 25 | self.assertEqual(self.environment.existing_resources[0].name, "db1") 26 | self.assertEqual(self.environment.existing_resources[0].instance_name, "foobar") 27 | 28 | 29 | class EnvironmentBothUseExistingDBTestCase(EnvironmentBaseTestCase): 30 | fixture_name = "backend-1-service-1-db-both-use-existing" 31 | 32 | def test_environment_use_existing_db(self): 33 | self.assertEqual(len(self.collection.environments), 1) 34 | self.assertEqual(self.environment.existing_resources[0].name, "db1") 35 | self.assertEqual(self.environment.existing_resources[0].instance_name, "bazbar") 36 | 37 | 38 | class EnvironmentCollectionTwoServicesandDBsTwoEnvsTest(EnvironmentBaseTestCase): 39 | fixture_name = "backend-1-service-2-db-2-envs" 40 | 41 | def test_environment_domain(self): 42 | self.assertEqual(len(self.collection.environments), 2) 43 | self.assertEqual( 44 | self.environment.domain, 45 | "main.my-backend-test-app.testnewsite.coherencesites.com", 46 | ) 47 | 48 | def test_environment_services(self): 49 | self.assertEqual(len(self.environment.services), 3) 50 | self.assertEqual(len(self.environment.web_services), 1) 51 | -------------------------------------------------------------------------------- /src/cnc/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .string import clean_name_string 2 | -------------------------------------------------------------------------------- /src/cnc/utils/string.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def replace_all(text): 5 | # Replace all non word characters 6 | text = text.replace(" ", "") 7 | return re.sub("\\W|_", "-", text) 8 | 9 | 10 | def remove_multiple_hyphens(string): 11 | # make sure no double hyphens, AWS won't allow 12 | cleaned = re.sub(r"-+", "-", string) 13 | return cleaned 14 | 15 | 16 | def clean_name_string(raw_string, truncate_len=40): 17 | cleaned = replace_all(raw_string).lower() 18 | cleaned = cleaned.strip("-") 19 | cleaned = remove_multiple_hyphens(cleaned) 20 | 21 | if len(cleaned) > truncate_len: 22 | truncated = cleaned[:truncate_len].strip("-") 23 | else: 24 | truncated = cleaned 25 | 26 | return truncated 27 | --------------------------------------------------------------------------------