├── .github ├── cdk │ ├── .gitignore │ ├── cdkactions.yaml │ ├── docker.ts │ ├── main.ts │ ├── package.json │ ├── tsconfig.json │ └── yarn.lock ├── renovate.json └── workflows │ ├── cdkactions_docker-django-base.yaml │ ├── cdkactions_docker-pg-s3-backup.yaml │ ├── cdkactions_docker-shibboleth-sp-nginx.yaml │ ├── cdkactions_docker-team-sync.yaml │ ├── cdkactions_docker-waypoint.yaml │ ├── cdkactions_kittyhawk.yaml │ ├── cdkactions_kraken.yaml │ ├── cdkactions_terraform.yaml │ ├── pg-s3-backup.yaml │ └── release-waypoint.yaml ├── .gitignore ├── LICENSE ├── README.md ├── cdk ├── README.md ├── kittyhawk │ ├── .eslintrc.json │ ├── .gitattributes │ ├── .gitignore │ ├── .npmignore │ ├── .prettierignore │ ├── .prettierrc.json │ ├── .projen │ │ ├── deps.json │ │ ├── files.json │ │ └── tasks.json │ ├── .projenrc.js │ ├── API.md │ ├── CHANGELOG.md │ ├── LICENSE │ ├── README.md │ ├── cdk8s.yaml │ ├── package.json │ ├── src │ │ ├── application │ │ │ ├── base.ts │ │ │ ├── django.ts │ │ │ ├── index.ts │ │ │ ├── react.ts │ │ │ └── redis.ts │ │ ├── certificate.ts │ │ ├── chart.ts │ │ ├── container.ts │ │ ├── cronjob.ts │ │ ├── deployment.ts │ │ ├── imports │ │ │ ├── acme.cert-manager.io.ts │ │ │ ├── cert-manager.io.ts │ │ │ └── k8s.ts │ │ ├── index.ts │ │ ├── ingress.ts │ │ ├── service.ts │ │ ├── serviceaccount.ts │ │ └── utils.ts │ ├── test │ │ ├── __snapshots__ │ │ │ ├── certificate.test.ts.snap │ │ │ ├── cronjob.test.ts.snap │ │ │ ├── deployment.test.ts.snap │ │ │ └── ingress.test.ts.snap │ │ ├── application │ │ │ ├── __snapshots__ │ │ │ │ ├── application.test.ts.snap │ │ │ │ ├── django.test.ts.snap │ │ │ │ ├── react.test.ts.snap │ │ │ │ └── redis.test.ts.snap │ │ │ ├── application.test.ts │ │ │ ├── django.test.ts │ │ │ ├── react.test.ts │ │ │ └── redis.test.ts │ │ ├── certificate.test.ts │ │ ├── cronjob.test.ts │ │ ├── deployment.test.ts │ │ ├── ingress.test.ts │ │ └── utils.ts │ ├── tsconfig.dev.json │ ├── tsconfig.eslint.json │ ├── tsconfig.jest.json │ ├── tsconfig.json │ └── yarn.lock ├── kraken │ ├── .eslintrc.json │ ├── .gitattributes │ ├── .gitignore │ ├── .npmignore │ ├── .prettierignore │ ├── .prettierrc.json │ ├── .projen │ │ ├── deps.json │ │ ├── files.json │ │ └── tasks.json │ ├── .projenrc.js │ ├── .versionrc.json │ ├── CHANGELOG.md │ ├── LICENSE │ ├── README.md │ ├── package.json │ ├── src │ │ ├── auto-approve.ts │ │ ├── cdk.ts │ │ ├── deploy.ts │ │ ├── django-project.ts │ │ ├── django.ts │ │ ├── docker.ts │ │ ├── index.ts │ │ ├── integration-tests.ts │ │ ├── labs-application.ts │ │ ├── postintegrationimagepublishjob.ts │ │ ├── pypi.ts │ │ ├── react-project.ts │ │ ├── react.ts │ │ └── utils.ts │ ├── test │ │ ├── __snapshots__ │ │ │ ├── auto-approve.test.ts.snap │ │ │ ├── cdk.test.ts.snap │ │ │ ├── custom.test.ts.snap │ │ │ ├── django-project.test.ts.snap │ │ │ ├── django.test.ts.snap │ │ │ ├── docker.test.ts.snap │ │ │ ├── integration-tests.test.ts.snap │ │ │ ├── labs-application.test.ts.snap │ │ │ ├── pypi.test.ts.snap │ │ │ ├── react-project.test.ts.snap │ │ │ └── react.test.ts.snap │ │ ├── auto-approve.test.ts │ │ ├── cdk.test.ts │ │ ├── custom.test.ts │ │ ├── django-project.test.ts │ │ ├── django.test.ts │ │ ├── docker.test.ts │ │ ├── integration-tests.test.ts │ │ ├── labs-application.test.ts │ │ ├── pypi.test.ts │ │ ├── react-project.test.ts │ │ ├── react.test.ts │ │ ├── utils.test.ts │ │ └── utils.ts │ ├── tsconfig.dev.json │ ├── tsconfig.jest.json │ ├── tsconfig.json │ ├── version.json │ └── yarn.lock └── projen-common.js ├── docker ├── README.md ├── django-base │ ├── .dockerignore │ ├── Dockerfile │ ├── LICENSE │ ├── README.md │ ├── django-run │ └── mime.types ├── pg-s3-backup │ ├── Dockerfile │ ├── README.md │ └── backup ├── shibboleth-sp-nginx │ ├── .dockerignore │ ├── Dockerfile │ ├── LICENSE │ ├── README.md │ ├── VERSION.txt │ ├── nginx-default.conf │ ├── nginx │ │ ├── cert.pem │ │ ├── key.pem │ │ ├── nginx.conf │ │ └── shib_clear_headers │ ├── shibd.logger │ └── supervisord.conf ├── team-sync │ ├── .dockerignore │ ├── .gitignore │ ├── Dockerfile │ ├── LICENSE │ ├── Pipfile │ ├── Pipfile.lock │ ├── README.md │ ├── setup.cfg │ └── sync │ │ ├── __init__.py │ │ ├── modules │ │ ├── __init__.py │ │ ├── bitwarden.py │ │ ├── platform.py │ │ ├── user-policy.hcl.j2 │ │ └── vault.py │ │ └── sync.py └── waypoint │ ├── Dockerfile │ ├── README.md │ ├── database-init │ ├── install.sh │ ├── setup.py │ ├── src │ ├── __init__.py │ ├── main.py │ └── waypoint_client.py │ ├── utils │ └── courses_reset_db.sql │ ├── waypoint-bashrc │ └── waypoint-init ├── docs └── secrets │ ├── Github.md │ ├── README.md │ ├── Vault.md │ └── img │ ├── secrets-flow-dark.png │ └── secrets-flow-light.png ├── grafana-dashboards ├── README.md ├── cert-manager.json ├── pod-alerting-dashboard.json ├── pod-dashboard.json └── traefik.json ├── renovate ├── README.md └── renovate.json └── terraform ├── .terraform.lock.hcl ├── Bootstrapping.md ├── README.md ├── bastion.tf ├── db-backup.tf ├── eks.tf ├── files ├── bastion │ ├── container_exec.sh │ ├── container_exec_entry.sh │ ├── ssh_authorized_keys │ └── user_data.sh ├── kubeconfig └── vault_user_data.sh ├── gh-actions.tf ├── github.tf ├── helm ├── aws-node-termination-handler.yaml ├── bitwarden.yaml ├── cert-manager.yaml ├── datadog.yaml ├── db-backup.yaml ├── grafana.yaml ├── ingress │ ├── bitwarden-ingress.yaml │ ├── expander-expander.yaml │ ├── penn-basics-react.yaml │ ├── penn-clubs-hub-django-wsgi.yaml │ ├── penn-clubs-hub-react.yaml │ └── platform-dev.yaml ├── prometheus.yaml ├── team-sync.yaml ├── traefik.yaml └── vault-secret-sync.yaml ├── iam.tf ├── main.tf ├── modules ├── README.md ├── base_cluster │ ├── README.md │ ├── cert-manager.tf │ ├── clusterissuer.yaml │ ├── datadog.tf │ ├── main.tf │ ├── monitoring.tf │ ├── redis.tf │ ├── traefik.tf │ ├── variables.tf │ ├── vault-secret-sync.tf │ └── wildcard-cert.yaml ├── domain │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── iam │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── vault │ ├── README.md │ ├── grafana.tf │ ├── main.tf │ ├── outputs.tf │ ├── policies │ │ ├── admin.hcl │ │ ├── secret-sync.hcl │ │ └── team-sync.hcl │ ├── secret-sync.tf │ ├── team-sync.tf │ └── variables.tf └── vault_flush │ ├── README.md │ ├── main.tf │ └── variables.tf ├── production-cluster.tf ├── provider.tf ├── rds.tf ├── route53.tf ├── sre.tf ├── variables.tf ├── vault.tf └── vpc.tf /.github/cdk/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | *.js 3 | *.d.ts 4 | -------------------------------------------------------------------------------- /.github/cdk/cdkactions.yaml: -------------------------------------------------------------------------------- 1 | language: typescript 2 | app: node main.js 3 | -------------------------------------------------------------------------------- /.github/cdk/main.ts: -------------------------------------------------------------------------------- 1 | import { App, CheckoutJob, Stack, Workflow } from "cdkactions" 2 | import { CDKPublishStack } from "@pennlabs/kraken" 3 | import { Construct } from "constructs" 4 | import { 5 | DjangoBaseDockerStack, 6 | DockerPublishStack, 7 | ShibbolethDockerStack, 8 | } from "./docker" 9 | 10 | class TerraformLintStack extends Stack { 11 | constructor(scope: Construct, name: string) { 12 | super(scope, name) 13 | 14 | const workflow = new Workflow(this, "terraform", { 15 | name: "Lint terraform files", 16 | on: { 17 | push: { 18 | paths: ["terraform/**.tf"], 19 | }, 20 | }, 21 | }) 22 | 23 | new CheckoutJob(workflow, "lint", { 24 | runsOn: "ubuntu-latest", 25 | steps: [ 26 | { 27 | uses: "hashicorp/setup-terraform@v1", 28 | }, 29 | { 30 | run: "terraform fmt -check -recursive terraform", 31 | }, 32 | ], 33 | }) 34 | } 35 | } 36 | 37 | const app = new App() 38 | 39 | // CDK stacks 40 | new CDKPublishStack(app, "kraken") 41 | new CDKPublishStack(app, "kittyhawk") 42 | 43 | // Docker stacks 44 | const dockerImages = ["datadog-agent", "pg-s3-backup", "team-sync"] 45 | dockerImages.map((name) => new DockerPublishStack(app, name)) 46 | new DjangoBaseDockerStack(app, { 47 | pythonVersions: ["3.9.14", "3.8.5", "3.10.1", "3.11"], 48 | }) 49 | new ShibbolethDockerStack(app) 50 | 51 | // Misc stacks 52 | new TerraformLintStack(app, "terraform") 53 | app.synth() 54 | -------------------------------------------------------------------------------- /.github/cdk/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdk", 3 | "version": "0.1.0", 4 | "main": "main.js", 5 | "types": "main.d.ts", 6 | "license": "MIT", 7 | "private": true, 8 | "scripts": { 9 | "synth": "cdkactions synth", 10 | "compile": "tsc", 11 | "watch": "tsc -w", 12 | "build": "yarn compile && yarn synth", 13 | "upgrade-cdk": "yarn upgrade cdkactions@latest cdkactions-cli@latest" 14 | }, 15 | "dependencies": { 16 | "@pennlabs/kraken": "^0.8.10", 17 | "cdkactions": "^0.2.3", 18 | "constructs": "^3.3.259" 19 | }, 20 | "devDependencies": { 21 | "@types/node": "^17.0.23", 22 | "cdkactions-cli": "^0.2.3", 23 | "typescript": "^4.6.3" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /.github/cdk/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "alwaysStrict": true, 4 | "charset": "utf8", 5 | "declaration": true, 6 | "experimentalDecorators": true, 7 | "inlineSourceMap": true, 8 | "inlineSources": true, 9 | "lib": [ 10 | "es2018" 11 | ], 12 | "module": "CommonJS", 13 | "noEmitOnError": true, 14 | "noFallthroughCasesInSwitch": true, 15 | "noImplicitAny": true, 16 | "noImplicitReturns": true, 17 | "noImplicitThis": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "resolveJsonModule": true, 21 | "strict": true, 22 | "strictNullChecks": true, 23 | "strictPropertyInitialization": true, 24 | "stripInternal": true, 25 | "target": "ES2018" 26 | }, 27 | "include": [ 28 | "**/*.ts" 29 | ], 30 | "exclude": [ 31 | "node_modules" 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "github>pennlabs/infrastructure//renovate/renovate" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /.github/workflows/cdkactions_docker-pg-s3-backup.yaml: -------------------------------------------------------------------------------- 1 | # Generated by cdkactions. Do not modify 2 | # Generated as part of the 'pg-s3-backup' stack. 3 | name: Publish pg-s3-backup 4 | on: 5 | push: 6 | paths: 7 | - docker/pg-s3-backup/** 8 | jobs: 9 | publish-pg-s3-backup: 10 | name: Publish pg-s3-backup 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: docker/setup-qemu-action@v3 15 | - uses: docker/setup-buildx-action@v3 16 | - name: Cache Docker layers 17 | uses: actions/cache@v4 18 | with: 19 | path: /tmp/.buildx-cache 20 | key: buildx-publish-pg-s3-backup 21 | - uses: docker/login-action@v3 22 | with: 23 | username: ${{ secrets.DOCKER_USERNAME }} 24 | password: ${{ secrets.DOCKER_PASSWORD }} 25 | - name: Build/Publish 26 | uses: docker/build-push-action@v6 27 | with: 28 | context: docker/pg-s3-backup 29 | file: docker/pg-s3-backup/Dockerfile 30 | push: ${{ github.ref == 'refs/heads/master' }} 31 | cache-from: type=local,src=/tmp/.buildx-cache,type=registry,ref=pennlabs/pg-s3-backup:latest 32 | cache-to: type=local,dest=/tmp/.buildx-cache 33 | tags: pennlabs/pg-s3-backup:latest,pennlabs/pg-s3-backup:${{ github.sha }} 34 | -------------------------------------------------------------------------------- /.github/workflows/cdkactions_docker-shibboleth-sp-nginx.yaml: -------------------------------------------------------------------------------- 1 | # Generated by cdkactions. Do not modify 2 | # Generated as part of the 'shibboleth' stack. 3 | name: Publish shibboleth-sp-nginx 4 | on: 5 | push: 6 | paths: 7 | - docker/shibboleth-sp-nginx/** 8 | jobs: 9 | version: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - id: version 14 | name: Set version 15 | run: echo "::set-output name=version::$(cat docker/shibboleth-sp-nginx/VERSION.txt)" 16 | outputs: 17 | version: ${{ steps.version.outputs.version }} 18 | publish-shibboleth-sp-nginx: 19 | name: Publish shibboleth-sp-nginx 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@v4 23 | - uses: docker/setup-qemu-action@v3 24 | - uses: docker/setup-buildx-action@v3 25 | - name: Cache Docker layers 26 | uses: actions/cache@v4 27 | with: 28 | path: /tmp/.buildx-cache 29 | key: buildx-publish-shibboleth-sp-nginx 30 | - uses: docker/login-action@v3 31 | with: 32 | username: ${{ secrets.DOCKER_USERNAME }} 33 | password: ${{ secrets.DOCKER_PASSWORD }} 34 | - name: Build/Publish 35 | uses: docker/build-push-action@v6 36 | with: 37 | context: docker/shibboleth-sp-nginx 38 | file: docker/shibboleth-sp-nginx/Dockerfile 39 | push: ${{ github.ref == 'refs/heads/master' }} 40 | cache-from: type=local,src=/tmp/.buildx-cache,type=registry,ref=pennlabs/shibboleth-sp-nginx:latest 41 | cache-to: type=local,dest=/tmp/.buildx-cache 42 | tags: pennlabs/shibboleth-sp-nginx:latest,pennlabs/shibboleth-sp-nginx:${{ needs.version.outputs.version }} 43 | needs: version 44 | -------------------------------------------------------------------------------- /.github/workflows/cdkactions_docker-team-sync.yaml: -------------------------------------------------------------------------------- 1 | # Generated by cdkactions. Do not modify 2 | # Generated as part of the 'team-sync' stack. 3 | name: Publish team-sync 4 | on: 5 | push: 6 | paths: 7 | - docker/team-sync/** 8 | jobs: 9 | publish-team-sync: 10 | name: Publish team-sync 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: docker/setup-qemu-action@v3 15 | - uses: docker/setup-buildx-action@v3 16 | - name: Cache Docker layers 17 | uses: actions/cache@v4 18 | with: 19 | path: /tmp/.buildx-cache 20 | key: buildx-publish-team-sync 21 | - uses: docker/login-action@v3 22 | with: 23 | username: ${{ secrets.DOCKER_USERNAME }} 24 | password: ${{ secrets.DOCKER_PASSWORD }} 25 | - name: Build/Publish 26 | uses: docker/build-push-action@v6 27 | with: 28 | context: docker/team-sync 29 | file: docker/team-sync/Dockerfile 30 | push: ${{ github.ref == 'refs/heads/master' }} 31 | cache-from: type=local,src=/tmp/.buildx-cache,type=registry,ref=pennlabs/team-sync:latest 32 | cache-to: type=local,dest=/tmp/.buildx-cache 33 | tags: pennlabs/team-sync:latest,pennlabs/team-sync:${{ github.sha }} 34 | -------------------------------------------------------------------------------- /.github/workflows/cdkactions_docker-waypoint.yaml: -------------------------------------------------------------------------------- 1 | name: Publish waypoint 2 | on: 3 | push: 4 | paths: 5 | - docker/waypoint/** 6 | 7 | jobs: 8 | publish-waypoint: 9 | name: Publish waypoint 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: docker/setup-qemu-action@v3 14 | with: 15 | platforms: linux/amd64,linux/arm64 16 | image: tonistiigi/binfmt:latest 17 | - uses: docker/setup-buildx-action@v3 18 | with: 19 | platforms: linux/amd64,linux/arm64 20 | - name: Cache Docker layers 21 | uses: actions/cache@v3 22 | with: 23 | path: /tmp/.buildx-cache 24 | key: buildx-publish-waypoint 25 | - uses: docker/login-action@v1 26 | with: 27 | username: ${{ secrets.DOCKER_USERNAME }} 28 | password: ${{ secrets.DOCKER_PASSWORD }} 29 | - name: Build/Publish 30 | uses: docker/build-push-action@v6 31 | with: 32 | context: docker/waypoint 33 | file: docker/waypoint/Dockerfile 34 | platforms: linux/amd64,linux/arm64 35 | push: true 36 | cache-from: type=local,src=/tmp/.buildx-cache,type=registry,ref=pennlabs/waypoint:latest 37 | cache-to: type=local,dest=/tmp/.buildx-cache 38 | tags: | 39 | pennlabs/waypoint:latest 40 | pennlabs/waypoint:${{ github.sha }} 41 | -------------------------------------------------------------------------------- /.github/workflows/cdkactions_kittyhawk.yaml: -------------------------------------------------------------------------------- 1 | # Generated by cdkactions. Do not modify 2 | # Generated as part of the 'kittyhawk' stack. 3 | name: Publish kittyhawk 4 | on: 5 | push: 6 | paths: 7 | - cdk/kittyhawk/** 8 | jobs: 9 | publish: 10 | runs-on: ubuntu-latest 11 | defaults: 12 | run: 13 | working-directory: cdk/kittyhawk 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Cache 17 | uses: actions/cache@v4 18 | with: 19 | path: "**/node_modules" 20 | key: v0-${{ hashFiles('cdk/kittyhawk/yarn.lock') }} 21 | - name: Install Dependencies 22 | run: yarn install --frozen-lockfile 23 | - name: Test 24 | run: yarn test 25 | - name: Upload Code Coverage 26 | run: |- 27 | yarn run codecov -p $GITHUB_WORKSPACE -F kittyhawk 28 | - name: Install jq 29 | run: |- 30 | curl -sSLo /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 31 | chmod +x /usr/bin/jq 32 | - name: Check if version is already published to npm 33 | id: version_check 34 | run: |- 35 | PACKAGE=$(node -p "require('./package.json').name") 36 | VERSION=$(node -p "require('./package.json').version") 37 | NEW_VERSION=$(yarn info $PACKAGE versions --json | jq ".data | any(. == \"$VERSION\") | not") 38 | echo "::set-output name=NEW_VERSION::$NEW_VERSION" 39 | - name: Publish to npm 40 | run: |- 41 | yarn compile 42 | yarn package 43 | mv dist/js/*.tgz dist/js/kittyhawk.tgz 44 | yarn publish --non-interactive --access public dist/js/kittyhawk.tgz 45 | if: github.ref == 'refs/heads/master' && steps.version_check.outputs.NEW_VERSION == 'true' 46 | env: 47 | NPM_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} 48 | - name: Build docs 49 | run: yarn docgen 50 | - name: Publish docs 51 | if: github.ref == 'refs/heads/master' 52 | uses: peaceiris/actions-gh-pages@v3 53 | with: 54 | personal_token: ${{ secrets.BOT_GITHUB_PAT }} 55 | external_repository: pennlabs/kittyhawk-docs 56 | cname: kittyhawk.pennlabs.org 57 | publish_branch: master 58 | publish_dir: cdk/kittyhawk/docs 59 | user_name: github-actions 60 | user_email: github-actions[bot]@users.noreply.github.com 61 | container: 62 | image: node:18 63 | -------------------------------------------------------------------------------- /.github/workflows/cdkactions_kraken.yaml: -------------------------------------------------------------------------------- 1 | # Generated by cdkactions. Do not modify 2 | # Generated as part of the 'kraken' stack. 3 | name: Publish kraken 4 | on: 5 | push: 6 | paths: 7 | - cdk/kraken/** 8 | jobs: 9 | publish: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - name: Cache 14 | uses: actions/cache@v4 15 | with: 16 | path: "**/node_modules" 17 | key: v0-${{ hashFiles('cdk/kraken/yarn.lock') }} 18 | - name: Install Dependencies 19 | run: |- 20 | cd cdk/kraken 21 | yarn install --frozen-lockfile 22 | - name: Test 23 | run: |- 24 | cd cdk/kraken 25 | yarn test 26 | - name: Upload Code Coverage 27 | run: |- 28 | ROOT=$(pwd) 29 | cd cdk/kraken 30 | yarn run codecov -p $ROOT -F kraken 31 | - name: Install jq 32 | run: |- 33 | curl -sSLo /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 34 | chmod +x /usr/bin/jq 35 | - name: Check if version is already published to npm 36 | id: version_check 37 | run: |- 38 | cd cdk/kraken 39 | PACKAGE=$(node -p "require('./package.json').name") 40 | VERSION=$(node -p "require('./package.json').version") 41 | NEW_VERSION=$(yarn info $PACKAGE versions --json | jq ".data | any(. == \"$VERSION\") | not") 42 | echo "::set-output name=NEW_VERSION::$NEW_VERSION" 43 | - name: Publish to npm 44 | run: |- 45 | cd cdk/kraken 46 | yarn compile 47 | yarn package 48 | mv dist/js/*.tgz dist/js/kraken.tgz 49 | yarn publish --non-interactive --access public dist/js/kraken.tgz 50 | if: github.ref == 'refs/heads/master' && steps.version_check.outputs.NEW_VERSION == 'true' 51 | env: 52 | NPM_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} 53 | - name: Build docs 54 | run: |- 55 | cd cdk/kraken 56 | yarn docgen 57 | - name: Publish docs 58 | if: github.ref == 'refs/heads/master' 59 | uses: peaceiris/actions-gh-pages@v3 60 | with: 61 | personal_token: ${{ secrets.BOT_GITHUB_PAT }} 62 | external_repository: pennlabs/kraken-docs 63 | cname: kraken.pennlabs.org 64 | publish_branch: master 65 | publish_dir: cdk/kraken/docs 66 | user_name: github-actions 67 | user_email: github-actions[bot]@users.noreply.github.com 68 | container: 69 | image: node:18 70 | -------------------------------------------------------------------------------- /.github/workflows/cdkactions_terraform.yaml: -------------------------------------------------------------------------------- 1 | # Generated by cdkactions. Do not modify 2 | # Generated as part of the 'terraform' stack. 3 | name: Lint terraform files 4 | on: 5 | push: 6 | paths: 7 | - terraform/**.tf 8 | jobs: 9 | lint: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: hashicorp/setup-terraform@v1 14 | - run: terraform fmt -check -recursive terraform 15 | -------------------------------------------------------------------------------- /.github/workflows/pg-s3-backup.yaml: -------------------------------------------------------------------------------- 1 | name: Publish pg-s3-backup 2 | on: 3 | push: 4 | paths: 5 | - "docker/pg-s3-backup/**" 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | - name: Publish pg-s3-backup 12 | uses: docker/build-push-action@v1 13 | with: 14 | repository: pennlabs/pg-s3-backup 15 | path: docker/pg-s3-backup 16 | username: ${{ secrets.DOCKER_USERNAME }} 17 | password: ${{ secrets.DOCKER_PASSWORD }} 18 | push: ${{ github.ref == 'refs/heads/master' }} 19 | tags: "latest,${{ github.sha }}" 20 | -------------------------------------------------------------------------------- /.github/workflows/release-waypoint.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | build: 10 | strategy: 11 | matrix: 12 | include: 13 | - os: ubuntu-22.04 14 | arch: x86_64 15 | binary_suffix: '-linux-x86_64' 16 | - os: macos-latest 17 | arch: x86_64 18 | binary_suffix: '-macos-x86_64' 19 | - os: macos-latest 20 | arch: arm64 21 | binary_suffix: '-macos-arm64' 22 | 23 | runs-on: ${{ matrix.os }} 24 | steps: 25 | - uses: actions/checkout@v3 26 | 27 | - name: Set up Python 28 | uses: actions/setup-python@v4 29 | with: 30 | python-version: '3.11' 31 | 32 | - name: Install dependencies 33 | run: | 34 | pip install pyinstaller 35 | cd docker/waypoint && pip install . 36 | 37 | - name: Build binary 38 | working-directory: docker/waypoint 39 | run: pyinstaller src/waypoint_client.py --onefile --name "waypoint-client${{ matrix.binary_suffix }}" 40 | 41 | - name: Calculate SHA256 42 | run: | 43 | cd docker/waypoint/dist 44 | echo "BINARY_SHA=$(sha256sum waypoint-client${{ matrix.binary_suffix }} | awk '{print $1}')" >> $GITHUB_ENV 45 | 46 | - name: Create Release 47 | uses: softprops/action-gh-release@v1 48 | with: 49 | files: docker/waypoint/dist/waypoint-client* 50 | name: Release ${{ github.ref_name }} 51 | body: | 52 | SHA256 (${{ matrix.os }}-${{ matrix.arch }}): ${{ env.BINARY_SHA }} 53 | env: 54 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | labs-kubeconfig.yaml 2 | kubeconfig.yaml 3 | db_creds.json 4 | creds.txt 5 | .terraform 6 | terraform.* 7 | tf_creds.json 8 | .DS_Store 9 | *.log 10 | 11 | __pycache__ 12 | waypoint.egg-info 13 | 14 | *.env -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Penn Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Infrastructure 2 | 3 | In this repo, you can find all the configs for Penn Labs' infrastructure. Each directory contains additional information about what it does. 4 | -------------------------------------------------------------------------------- /cdk/README.md: -------------------------------------------------------------------------------- 1 | # cdk 2 | 3 | This folder contains the cdk abstractions built for Penn Labs. 4 | 5 | We use [projen](https://github.com/projen/projen) to handle most of the configuration of the cdk abstractions. 6 | 7 | ## Setting up a new CDK project 8 | 9 | Create a new directory for the cdk and populate it with the following commands: 10 | 11 | ```bash 12 | mkdir $name 13 | cd $name 14 | npx projen new typescript 15 | ``` 16 | 17 | Then modify the generated `.projenrc.js` to use the shared `cdk/project-common.js`. See kraken for an example. 18 | 19 | Now add the following to the `main.ts` file in the infrastructure repo cdk `new CDKPublishStack(app, '$name');`. 20 | 21 | Finally, create a new repo named "$name-docs", make sure the pennlabsbot account has push access, and set up a CNAME so `$name.pennlabs.org` points to `pennlabs.github.io.` 22 | 23 | After the first version of the docs has been deployed, make sure to enable gh pages, enforce HTTPS, and configure branch protections. 24 | -------------------------------------------------------------------------------- /cdk/kittyhawk/.gitattributes: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "npx projen". 2 | 3 | *.snap linguist-generated 4 | /.eslintrc.json linguist-generated 5 | /.gitattributes linguist-generated 6 | /.gitignore linguist-generated 7 | /.npmignore linguist-generated 8 | /.prettierignore linguist-generated 9 | /.prettierrc.json linguist-generated 10 | /.projen/** linguist-generated 11 | /.projen/deps.json linguist-generated 12 | /.projen/files.json linguist-generated 13 | /.projen/tasks.json linguist-generated 14 | /LICENSE linguist-generated 15 | /package.json linguist-generated 16 | /tsconfig.dev.json linguist-generated 17 | /tsconfig.json linguist-generated 18 | /yarn.lock linguist-generated -------------------------------------------------------------------------------- /cdk/kittyhawk/.gitignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "npx projen". 2 | !/.gitattributes 3 | !/.projen/tasks.json 4 | !/.projen/deps.json 5 | !/.projen/files.json 6 | !/package.json 7 | !/LICENSE 8 | !/.npmignore 9 | logs 10 | *.log 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | lerna-debug.log* 15 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | lib-cov 21 | coverage 22 | *.lcov 23 | .nyc_output 24 | build/Release 25 | node_modules/ 26 | jspm_packages/ 27 | *.tsbuildinfo 28 | .eslintcache 29 | *.tgz 30 | .yarn-integrity 31 | .cache 32 | /docs 33 | !/.projenrc.js 34 | /test-reports/ 35 | junit.xml 36 | /coverage/ 37 | !/.prettierignore 38 | !/.prettierrc.json 39 | !/test/ 40 | !/tsconfig.json 41 | !/tsconfig.dev.json 42 | !/src/ 43 | /lib 44 | /dist/ 45 | !/.eslintrc.json 46 | -------------------------------------------------------------------------------- /cdk/kittyhawk/.npmignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "npx projen". 2 | /.projen/ 3 | /test-reports/ 4 | junit.xml 5 | /coverage/ 6 | /test/ 7 | /tsconfig.dev.json 8 | /src/ 9 | !/lib/ 10 | !/lib/**/*.js 11 | !/lib/**/*.d.ts 12 | dist 13 | /tsconfig.json 14 | /.github/ 15 | /.vscode/ 16 | /.idea/ 17 | /.projenrc.js 18 | tsconfig.tsbuildinfo 19 | /.eslintrc.json 20 | -------------------------------------------------------------------------------- /cdk/kittyhawk/.prettierignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "npx projen". 2 | src/imports 3 | -------------------------------------------------------------------------------- /cdk/kittyhawk/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "overrides": [] 3 | } 4 | -------------------------------------------------------------------------------- /cdk/kittyhawk/.projen/deps.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": [ 3 | { 4 | "name": "@types/jest", 5 | "type": "build" 6 | }, 7 | { 8 | "name": "@types/node", 9 | "version": "^12", 10 | "type": "build" 11 | }, 12 | { 13 | "name": "@typescript-eslint/eslint-plugin", 14 | "version": "^5", 15 | "type": "build" 16 | }, 17 | { 18 | "name": "@typescript-eslint/parser", 19 | "version": "^5", 20 | "type": "build" 21 | }, 22 | { 23 | "name": "codecov", 24 | "type": "build" 25 | }, 26 | { 27 | "name": "eslint-config-prettier", 28 | "type": "build" 29 | }, 30 | { 31 | "name": "eslint-import-resolver-node", 32 | "type": "build" 33 | }, 34 | { 35 | "name": "eslint-import-resolver-typescript", 36 | "type": "build" 37 | }, 38 | { 39 | "name": "eslint-plugin-import", 40 | "type": "build" 41 | }, 42 | { 43 | "name": "eslint-plugin-prettier", 44 | "type": "build" 45 | }, 46 | { 47 | "name": "eslint", 48 | "version": "^8", 49 | "type": "build" 50 | }, 51 | { 52 | "name": "jest", 53 | "type": "build" 54 | }, 55 | { 56 | "name": "jest-junit", 57 | "version": "^13", 58 | "type": "build" 59 | }, 60 | { 61 | "name": "json-schema", 62 | "type": "build" 63 | }, 64 | { 65 | "name": "prettier", 66 | "type": "build" 67 | }, 68 | { 69 | "name": "projen", 70 | "type": "build" 71 | }, 72 | { 73 | "name": "ts-jest", 74 | "type": "build" 75 | }, 76 | { 77 | "name": "typedoc", 78 | "version": "^0.21.4", 79 | "type": "build" 80 | }, 81 | { 82 | "name": "typescript", 83 | "type": "build" 84 | }, 85 | { 86 | "name": "cdk8s", 87 | "type": "runtime" 88 | }, 89 | { 90 | "name": "cdk8s-cli", 91 | "type": "runtime" 92 | }, 93 | { 94 | "name": "constructs", 95 | "type": "runtime" 96 | }, 97 | { 98 | "name": "cron-time-generator", 99 | "type": "runtime" 100 | } 101 | ], 102 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"npx projen\"." 103 | } 104 | -------------------------------------------------------------------------------- /cdk/kittyhawk/.projen/files.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [ 3 | ".eslintrc.json", 4 | ".gitattributes", 5 | ".gitignore", 6 | ".npmignore", 7 | ".prettierignore", 8 | ".prettierrc.json", 9 | ".projen/deps.json", 10 | ".projen/files.json", 11 | ".projen/tasks.json", 12 | "LICENSE", 13 | "tsconfig.dev.json", 14 | "tsconfig.json" 15 | ], 16 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"npx projen\"." 17 | } 18 | -------------------------------------------------------------------------------- /cdk/kittyhawk/.projenrc.js: -------------------------------------------------------------------------------- 1 | const { TypeScriptProject } = require("projen/lib/typescript"); 2 | const common = require("../projen-common"); 3 | 4 | const project = new TypeScriptProject({ 5 | name: "@pennlabs/kittyhawk", 6 | description: 7 | "Tool to generate Kubernetes YAML files using Typescript. Built for Penn Labs.", 8 | deps: ["cdk8s", "constructs", "cron-time-generator", "cdk8s-cli"], 9 | devDeps: ["codecov"], 10 | keywords: ["cdk", "yaml", "kubernetes", "constructs", "cdk8s"], 11 | homepage: "https://kittyhawk.pennlabs.org", 12 | repositoryDirectory: "cdk/kittyhawk", 13 | ...common.options, 14 | scripts: { 15 | import: "yarn run cdk8s import --output src/imports", 16 | }, 17 | }); 18 | 19 | project.addFields({ ["version"]: "1.1.11" }); 20 | project.prettier?.ignoreFile?.addPatterns("src/imports"); 21 | project.testTask.steps.forEach((step) => { 22 | if (step.exec) { 23 | step.exec = step.exec.replace(" --updateSnapshot", ""); 24 | } 25 | }); 26 | project.setScript( 27 | "test", 28 | "export GIT_SHA='TESTSHA' AWS_ACCOUNT_ID='TEST_AWS_ACCOUNT_ID' && npx projen test" 29 | ); 30 | project.synth(); 31 | -------------------------------------------------------------------------------- /cdk/kittyhawk/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.1.4 (2022-04-18) 4 | 5 | * Better handling of custom ports 6 | * Fix bug for ingress ports 7 | * Disallow custom ports within ingress prop in `Application` and its extended classes 8 | * Update tests & snapshots 9 | * Update `API.md` file 10 | 11 | ## 1.1.3 (2022-04-08) 12 | 13 | * Upgrade dependencies and fix projen version configuration 14 | 15 | ## 1.1.2 (2022-04-03) 16 | * Dependency updates 17 | * Move cdk8s-cli from `devDependencies` to `dependencies` 18 | * Modify dependencies to use latest 19 | 20 | ## 1.1.1 (2022-03-28) 21 | * Fix incorrect update snapshot bug during `yarn test` 22 | * Make `name` of `PennLabsChart` optional with default value "pennlabs" 23 | 24 | ## 1.1.0 (2022-03-23) 25 | 26 | * Initial release 27 | 28 | ## 1.0.7 (2021-02-05) 29 | 30 | * Added domain parsing for certificate generation, using the `isSubdomain` flag passed along with each domain. 31 | * Allow multiple domains in a Django application. 32 | * `domain` property has been renamed to `domains`. `domains` requires is now an array where each object has type `{host: string, isSubdomain: boolean}`. 33 | * If multiple domains are defined, the `DOMAIN` environment variable will contain a comma-separated list of domains. 34 | * Fix incorrect OHQ test, and updated other tests. 35 | -------------------------------------------------------------------------------- /cdk/kittyhawk/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2024 Penn Labs 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /cdk/kittyhawk/cdk8s.yaml: -------------------------------------------------------------------------------- 1 | language: typescript 2 | app: node main.js 3 | imports: 4 | - k8s@1.22.0 5 | - https://github.com/jetstack/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml 6 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/application/base.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { removeSubdomain } from ".."; 3 | import { Certificate } from "../certificate"; 4 | import { Deployment, DeploymentProps } from "../deployment"; 5 | import { Ingress, IngressProps } from "../ingress"; 6 | import { Service } from "../service"; 7 | import { ServiceAccount } from "../serviceaccount"; 8 | 9 | export interface ApplicationProps { 10 | /** 11 | * Ingress configuration 12 | */ 13 | readonly ingress?: Omit; 14 | 15 | /** 16 | * Deployment configuration 17 | */ 18 | readonly deployment: DeploymentProps; 19 | 20 | /** 21 | * Port to expose the application on. 22 | */ 23 | readonly port?: number; 24 | 25 | /** 26 | * Creates a service account and attach it to any deployment pods. 27 | * serviceAccountName: release name 28 | */ 29 | readonly createServiceAccount?: boolean; 30 | } 31 | 32 | export class Application extends Construct { 33 | constructor(scope: Construct, appname: string, props: ApplicationProps) { 34 | super(scope, appname); 35 | 36 | // We want to prepend the project name to the name of each component 37 | const release_name = process.env.RELEASE_NAME || "undefined_release"; 38 | const fullname = `${release_name}-${appname}`; 39 | 40 | new Service(this, fullname, props.port); 41 | 42 | if (props.createServiceAccount) { 43 | new ServiceAccount(this, `${appname}-${release_name}`, { 44 | serviceAccountName: release_name, 45 | }); 46 | } 47 | 48 | new Deployment(this, fullname, { 49 | ...props.deployment, 50 | port: props.port, 51 | ...(props.createServiceAccount 52 | ? { serviceAccountName: release_name } 53 | : {}), 54 | }); 55 | 56 | if (props.ingress) { 57 | new Ingress(this, fullname, { 58 | ...props.ingress, 59 | port: props.port, 60 | }); 61 | 62 | const alreadyCreatedCertificates = new Set(); 63 | 64 | props.ingress.rules.forEach((rules) => { 65 | const finalDomain: string = removeSubdomain( 66 | rules.host, 67 | rules.isSubdomain ?? false 68 | ); 69 | if (!alreadyCreatedCertificates.has(finalDomain)) { 70 | new Certificate(this, fullname, rules); 71 | alreadyCreatedCertificates.add(finalDomain); 72 | } 73 | }); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/application/django.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { HostRules } from ".."; 3 | import { DeploymentProps } from "../deployment"; 4 | import { IngressProps } from "../ingress"; 5 | import { NonEmptyArray, nonEmptyMap } from "../utils"; 6 | import { Application } from "./base"; 7 | 8 | export interface DjangoApplicationProps { 9 | /** 10 | * Deployment configuration 11 | */ 12 | readonly deployment: DeploymentProps; 13 | 14 | /** 15 | * DJANGO_SETTINGS_MODULE environment variable. 16 | */ 17 | readonly djangoSettingsModule: string; 18 | 19 | /** 20 | * Port to expose the application on. 21 | */ 22 | readonly port?: number; 23 | 24 | /** 25 | * Array of domain(s). 26 | * Host is the domain the application runs on, 27 | * and isSubdomain is true if the domain should be treated as a subdomain for certificate purposes. 28 | * paths is the list of paths to expose the application on. 29 | * See the certificate documentation for more details. 30 | * 31 | * Domain is optional if the application is not publicly accessible (e.g. celery) 32 | */ 33 | readonly domains?: NonEmptyArray; 34 | 35 | /** 36 | * Optional ingressProps to override the default ingress props. 37 | */ 38 | readonly ingressProps?: Partial>; 39 | 40 | /** 41 | * Creates a service account and attach it to any deployment pods. 42 | * serviceAccountName: release name 43 | */ 44 | readonly createServiceAccount?: boolean; 45 | } 46 | 47 | export class DjangoApplication extends Application { 48 | constructor( 49 | scope: Construct, 50 | appname: string, 51 | props: DjangoApplicationProps 52 | ) { 53 | // Now, we ensure there are no duplicate env variables, even if they redefine it 54 | const djangoExtraEnv = [ 55 | ...new Set([ 56 | ...(props.deployment?.env || []), 57 | ...(props.domains 58 | ? [ 59 | { 60 | name: "DOMAINS", 61 | value: nonEmptyMap(props.domains, (h) => h.host).join(), 62 | }, 63 | ] 64 | : []), 65 | { name: "DJANGO_SETTINGS_MODULE", value: props.djangoSettingsModule }, 66 | ]), 67 | ]; 68 | 69 | // If everything passes, construct the Application. 70 | super(scope, appname, { 71 | port: props.port ?? 80, 72 | deployment: { 73 | ...props.deployment, 74 | env: djangoExtraEnv, 75 | }, 76 | // Configure the ingress using paths if domains is defined. 77 | ingress: props.domains 78 | ? { 79 | rules: nonEmptyMap(props.domains, (h) => ({ 80 | host: h.host, 81 | paths: h.paths, 82 | isSubdomain: h.isSubdomain ?? false, 83 | })), 84 | ...props.ingressProps, 85 | } 86 | : undefined, 87 | createServiceAccount: props.createServiceAccount, 88 | }); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/application/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./base"; 2 | export * from "./django"; 3 | export * from "./react"; 4 | export * from "./redis"; 5 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/application/react.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { DeploymentProps } from "../deployment"; 3 | import { HostRules, IngressProps } from "../ingress"; 4 | import { Application } from "./base"; 5 | 6 | export interface ReactApplicationProps { 7 | /** 8 | * Deployment configuration 9 | */ 10 | readonly deployment: DeploymentProps; 11 | 12 | /** 13 | * Host is the domain the application runs on, 14 | * isSubdomain is true if the domain should be treated as a subdomain for certificate purposes. 15 | * paths is the list of paths to expose the application on. 16 | * See the certificate documentation for more details. 17 | * 18 | */ 19 | readonly domain: HostRules; 20 | 21 | /** 22 | * Port to expose the application on. 23 | */ 24 | readonly port?: number; 25 | 26 | /** 27 | * Optional ingressProps to override the default ingress props. 28 | */ 29 | readonly ingressProps?: Partial>; 30 | 31 | /** 32 | * Creates a service account and attach it to any deployment pods. 33 | * serviceAccountName: release name 34 | */ 35 | readonly createServiceAccount?: boolean; 36 | } 37 | 38 | export class ReactApplication extends Application { 39 | constructor(scope: Construct, appname: string, props: ReactApplicationProps) { 40 | // Now, we ensure there are no duplicate env variables, even if they redefine it 41 | const reactExtraEnv = [ 42 | ...new Set([ 43 | ...(props.deployment?.env || []), 44 | { name: "DOMAIN", value: props.domain.host }, 45 | { name: "PORT", value: props.port ? `${props.port}` : "80" }, 46 | ]), 47 | ]; 48 | 49 | // If everything passes, construct the Application. 50 | super(scope, appname, { 51 | port: props.port ?? 80, 52 | deployment: { 53 | ...props.deployment, 54 | env: reactExtraEnv, 55 | }, 56 | ingress: { 57 | rules: [props.domain], 58 | ...props.ingressProps, 59 | }, 60 | createServiceAccount: props.createServiceAccount, 61 | }); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/certificate.ts: -------------------------------------------------------------------------------- 1 | import { JsonPatch } from "cdk8s"; 2 | import { Construct } from "constructs"; 3 | import { Certificate as CertApiObject } from "./imports/cert-manager.io"; 4 | import { domainToCertName, removeSubdomain, HostRules } from "./ingress"; 5 | import { defaultChildName } from "./utils"; 6 | 7 | const CERT_ISSUER_NAME = "wildcard-letsencrypt-prod"; 8 | 9 | export class Certificate extends Construct { 10 | constructor(scope: Construct, appname: string, rules: HostRules) { 11 | const hostString: string = domainToCertName( 12 | rules.host, 13 | rules.isSubdomain ?? false 14 | ); 15 | const finalDomain: string = removeSubdomain( 16 | rules.host, 17 | rules.isSubdomain ?? false 18 | ); 19 | 20 | super(scope, `certificate-${appname}-${hostString}`); 21 | 22 | const certificate = new CertApiObject(this, defaultChildName, { 23 | metadata: { 24 | name: hostString, 25 | labels: { 26 | "app.kubernetes.io/name": hostString, 27 | "app.kubernetes.io/component": "certificate", 28 | }, 29 | }, 30 | spec: { 31 | secretName: `${hostString}-tls`, 32 | dnsNames: [`${finalDomain}`, `*.${finalDomain}`], 33 | issuerRef: { 34 | name: CERT_ISSUER_NAME, 35 | kind: "ClusterIssuer", 36 | group: "cert-manager.io", 37 | }, 38 | }, 39 | }); 40 | 41 | // Remove labels added by PennLabsChart 42 | // ~1 is an escaped version of a forward slash "/" 43 | if (certificate.metadata.getLabel("app.kubernetes.io/part-of")) { 44 | certificate.addJsonPatch( 45 | JsonPatch.remove("/metadata/labels/app.kubernetes.io~1part-of") 46 | ); 47 | } 48 | if (certificate.metadata.getLabel("app.kubernetes.io/version")) { 49 | certificate.addJsonPatch( 50 | JsonPatch.remove("/metadata/labels/app.kubernetes.io~1version") 51 | ); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/chart.ts: -------------------------------------------------------------------------------- 1 | import { Chart, ChartProps } from "cdk8s"; 2 | import { Construct } from "constructs"; 3 | 4 | export class PennLabsChart extends Chart { 5 | constructor(scope: Construct, name?: string, props?: ChartProps) { 6 | const GIT_SHA = process.env.GIT_SHA; 7 | if (!GIT_SHA) { 8 | console.error("No GIT_SHA environment variable provided."); 9 | process.exit(1); 10 | } 11 | 12 | super(scope, name ?? "pennlabs", { 13 | namespace: props?.namespace, 14 | labels: { 15 | "app.kubernetes.io/part-of": 16 | process.env.RELEASE_NAME || "undefined_release", 17 | "app.kubernetes.io/version": GIT_SHA, 18 | "app.kubernetes.io/managed-by": "kittyhawk", 19 | ...props?.labels, 20 | }, 21 | }); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/deployment.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { Container, ContainerProps, SecretVolume } from "./container"; 3 | import { 4 | IntOrString, 5 | KubeDeployment as DeploymentApiObject, 6 | KubeServiceAccount, 7 | VolumeMount, 8 | Volume, 9 | } from "./imports/k8s"; 10 | import { defaultChildName } from "./utils"; 11 | 12 | export interface DeploymentProps extends ContainerProps { 13 | /** 14 | * Number of replicas to start. 15 | * 16 | * @default 1 17 | */ 18 | readonly replicas?: number; 19 | 20 | /** 21 | * Secret volume mounts for deployment container. 22 | * 23 | * @default [] 24 | */ 25 | readonly secretMounts?: VolumeMount[]; 26 | 27 | /** 28 | * Volume mounts for deployment container. 29 | * 30 | * This appends to the existing list of volumes, if created by the `secretMounts` property. 31 | * 32 | * @default [] 33 | */ 34 | readonly volumeMounts?: VolumeMount[]; 35 | 36 | /** 37 | * Volume mounts for deployment container. 38 | */ 39 | readonly volumes?: Volume[]; 40 | 41 | /** 42 | * The service account to be used to attach to any deployment pods. 43 | * Default serviceAccountName: release name 44 | */ 45 | readonly serviceAccount?: KubeServiceAccount; 46 | } 47 | 48 | export class Deployment extends Construct { 49 | constructor(scope: Construct, appname: string, props: DeploymentProps) { 50 | super(scope, `deployment-${appname}`); 51 | 52 | const label = { "app.kubernetes.io/name": appname }; 53 | const containers: Container[] = [new Container(props)]; 54 | const secretVolumes: SecretVolume[] = 55 | props.secretMounts?.map((m) => new SecretVolume(m)) || []; 56 | 57 | new DeploymentApiObject(this, defaultChildName, { 58 | metadata: { 59 | name: appname, 60 | labels: label, 61 | }, 62 | spec: { 63 | replicas: props.replicas ?? 1, 64 | selector: { 65 | matchLabels: label, 66 | }, 67 | strategy: { 68 | type: "RollingUpdate", 69 | rollingUpdate: { 70 | maxSurge: IntOrString.fromNumber(3), 71 | maxUnavailable: IntOrString.fromNumber(0), 72 | }, 73 | }, 74 | template: { 75 | metadata: { labels: label }, 76 | spec: { 77 | // The next line checks if serviceAccount exists, and adds it to serviceAccountName 78 | ...(props.serviceAccount 79 | ? { serviceAccountName: props.serviceAccount.name } 80 | : {}), 81 | containers: containers, 82 | volumes: [...secretVolumes, ...(props.volumes || [])], 83 | }, 84 | }, 85 | }, 86 | }); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./application"; 2 | export * from "./certificate"; 3 | export * from "./chart"; 4 | export * from "./container"; 5 | export * from "./cronjob"; 6 | export * from "./deployment"; 7 | export * from "./ingress"; 8 | export * from "./service"; 9 | export * from "./utils"; 10 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/service.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { KubeService as ServiceApiObject, IntOrString } from "./imports/k8s"; 3 | import { defaultChildName } from "./utils"; 4 | 5 | export class Service extends Construct { 6 | constructor(scope: Construct, appname: string, port?: number) { 7 | super(scope, `service-${appname}`); 8 | 9 | const targetPort = port ?? 80; 10 | 11 | new ServiceApiObject(this, defaultChildName, { 12 | metadata: { 13 | name: appname, 14 | labels: { "app.kubernetes.io/name": appname }, 15 | }, 16 | spec: { 17 | type: "ClusterIP", 18 | ports: [ 19 | { port: targetPort, targetPort: IntOrString.fromNumber(targetPort) }, 20 | ], 21 | selector: { "app.kubernetes.io/name": appname }, 22 | }, 23 | }); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/serviceaccount.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { KubeServiceAccount } from "./imports/k8s"; 3 | import { defaultChildName } from "./utils"; 4 | 5 | export interface ServiceAccountProps { 6 | readonly serviceAccountName: string; 7 | } 8 | 9 | export class ServiceAccount extends Construct { 10 | constructor(scope: Construct, appname: string, props: ServiceAccountProps) { 11 | super(scope, appname); 12 | 13 | const awsAccountId = process.env.AWS_ACCOUNT_ID; 14 | if (!awsAccountId) { 15 | console.error("No AWS_ACCOUNT_ID environment variable provided."); 16 | process.exit(1); 17 | } 18 | 19 | new KubeServiceAccount(this, defaultChildName, { 20 | metadata: { 21 | name: props.serviceAccountName, 22 | annotations: { 23 | ["eks.amazonaws.com/role-arn"]: `arn:aws:iam::${awsAccountId}:role/${props.serviceAccountName}`, 24 | }, 25 | }, 26 | }); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /cdk/kittyhawk/src/utils.ts: -------------------------------------------------------------------------------- 1 | export type NonEmptyArray = [T, ...T[]]; 2 | 3 | export function nonEmptyMap( 4 | arr: NonEmptyArray, 5 | callbackfn: (value: T) => U 6 | ): NonEmptyArray { 7 | return arr.map(callbackfn) as any as NonEmptyArray; 8 | } 9 | 10 | export const defaultChildName = "Default"; 11 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/__snapshots__/certificate.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`Certificate (Non Penn Labs Chart) -- Success 1`] = ` 4 | [ 5 | { 6 | "apiVersion": "cert-manager.io/v1", 7 | "kind": "Certificate", 8 | "metadata": { 9 | "labels": { 10 | "app.kubernetes.io/component": "certificate", 11 | "app.kubernetes.io/name": "pennlabs-org", 12 | }, 13 | "name": "pennlabs-org", 14 | }, 15 | "spec": { 16 | "dnsNames": [ 17 | "pennlabs.org", 18 | "*.pennlabs.org", 19 | ], 20 | "issuerRef": { 21 | "group": "cert-manager.io", 22 | "kind": "ClusterIssuer", 23 | "name": "wildcard-letsencrypt-prod", 24 | }, 25 | "secretName": "pennlabs-org-tls", 26 | }, 27 | }, 28 | ] 29 | `; 30 | 31 | exports[`Certificate -- Success 1`] = ` 32 | [ 33 | { 34 | "apiVersion": "cert-manager.io/v1", 35 | "kind": "Certificate", 36 | "metadata": { 37 | "labels": { 38 | "app.kubernetes.io/component": "certificate", 39 | "app.kubernetes.io/managed-by": "kittyhawk", 40 | "app.kubernetes.io/name": "pennlabs-org", 41 | }, 42 | "name": "pennlabs-org", 43 | }, 44 | "spec": { 45 | "dnsNames": [ 46 | "pennlabs.org", 47 | "*.pennlabs.org", 48 | ], 49 | "issuerRef": { 50 | "group": "cert-manager.io", 51 | "kind": "ClusterIssuer", 52 | "name": "wildcard-letsencrypt-prod", 53 | }, 54 | "secretName": "pennlabs-org-tls", 55 | }, 56 | }, 57 | ] 58 | `; 59 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/application/__snapshots__/application.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`Tag Override 1`] = ` 4 | Array [ 5 | Object { 6 | "apiVersion": "v1", 7 | "kind": "Service", 8 | "metadata": Object { 9 | "labels": Object { 10 | "app.kubernetes.io/managed-by": "kittyhawk", 11 | "app.kubernetes.io/name": "RELEASE_NAME-serve", 12 | "app.kubernetes.io/part-of": "RELEASE_NAME", 13 | "app.kubernetes.io/version": "TAG_FROM_CI", 14 | }, 15 | "name": "RELEASE_NAME-serve", 16 | }, 17 | "spec": Object { 18 | "ports": Array [ 19 | Object { 20 | "port": 80, 21 | "targetPort": 80, 22 | }, 23 | ], 24 | "selector": Object { 25 | "app.kubernetes.io/name": "RELEASE_NAME-serve", 26 | }, 27 | "type": "ClusterIP", 28 | }, 29 | }, 30 | Object { 31 | "apiVersion": "apps/v1", 32 | "kind": "Deployment", 33 | "metadata": Object { 34 | "labels": Object { 35 | "app.kubernetes.io/managed-by": "kittyhawk", 36 | "app.kubernetes.io/name": "RELEASE_NAME-serve", 37 | "app.kubernetes.io/part-of": "RELEASE_NAME", 38 | "app.kubernetes.io/version": "TAG_FROM_CI", 39 | }, 40 | "name": "RELEASE_NAME-serve", 41 | }, 42 | "spec": Object { 43 | "replicas": 1, 44 | "selector": Object { 45 | "matchLabels": Object { 46 | "app.kubernetes.io/name": "RELEASE_NAME-serve", 47 | }, 48 | }, 49 | "strategy": Object { 50 | "rollingUpdate": Object { 51 | "maxSurge": 3, 52 | "maxUnavailable": 0, 53 | }, 54 | "type": "RollingUpdate", 55 | }, 56 | "template": Object { 57 | "metadata": Object { 58 | "labels": Object { 59 | "app.kubernetes.io/name": "RELEASE_NAME-serve", 60 | }, 61 | }, 62 | "spec": Object { 63 | "containers": Array [ 64 | Object { 65 | "env": Array [ 66 | Object { 67 | "name": "GIT_SHA", 68 | "value": "TAG_FROM_CI", 69 | }, 70 | ], 71 | "image": "pennlabs/website:latest", 72 | "imagePullPolicy": "IfNotPresent", 73 | "name": "worker", 74 | "ports": Array [ 75 | Object { 76 | "containerPort": 80, 77 | }, 78 | ], 79 | }, 80 | ], 81 | "volumes": Array [], 82 | }, 83 | }, 84 | }, 85 | }, 86 | ] 87 | `; 88 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/application/application.test.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { Application } from "../../src/application"; 3 | import { chartTest, failingTestNoGitSha } from "../utils"; 4 | 5 | export function buildTagOverrideChart(scope: Construct) { 6 | /** Overrides the image tag set as env var **/ 7 | new Application(scope, "serve", { 8 | deployment: { 9 | image: "pennlabs/website", 10 | tag: "latest", 11 | }, 12 | }); 13 | } 14 | export function buildSimpleChart(scope: Construct) { 15 | /** Overrides the image tag set as env var **/ 16 | new Application(scope, "serve", { 17 | deployment: { 18 | image: "pennlabs/website", 19 | }, 20 | }); 21 | } 22 | 23 | test("Application -- No Git Sha", () => failingTestNoGitSha(buildSimpleChart)); 24 | test("Tag Override", () => chartTest(buildTagOverrideChart)); 25 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/application/django.test.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { HostRules } from "../../src"; 3 | import { DjangoApplication } from "../../src/application"; 4 | import { NonEmptyArray } from "../../src/utils"; 5 | import { chartTest } from "../utils"; 6 | 7 | /** 8 | * Test Configuration for DjangoApplication 9 | * 10 | * default - assumes mostly default values for the configuration 11 | * example - sample parameters 12 | * undefinedDomains - domains is undefined, no ingress required (example: celery on courses backend) 13 | * duplicateEnv - env is defined twice, should not throw an error 14 | * */ 15 | const djangoTestConfig = { 16 | default: { 17 | deployment: { 18 | image: "pennlabs/platform", 19 | }, 20 | domains: [ 21 | { host: "platform.pennlabs.org", paths: ["/"] }, 22 | ] as NonEmptyArray, 23 | djangoSettingsModule: "Platform.settings.production", 24 | createServiceAccount: true, 25 | }, 26 | example: { 27 | deployment: { 28 | image: "pennlabs/platform", 29 | replicas: 2, 30 | env: [{ name: "SOME_ENV", value: "environment variables are cool" }], 31 | }, 32 | domains: [ 33 | { host: "platform.pennlabs.org", isSubdomain: true, paths: ["/"] }, 34 | ] as NonEmptyArray, 35 | djangoSettingsModule: "Platform.settings.production", 36 | port: 8080, 37 | }, 38 | duplicateEnv: { 39 | deployment: { 40 | image: "pennlabs/platform", 41 | env: [{ name: "DOMAIN", value: "platform.pennlabs.org" }], 42 | }, 43 | domains: [ 44 | { host: "platform.pennlabs.org", isSubdomain: true, paths: ["/"] }, 45 | ] as NonEmptyArray, 46 | djangoSettingsModule: "Platform.settings.production", 47 | }, 48 | undefinedDomains: { 49 | deployment: { 50 | image: "pennlabs/platform", 51 | env: [{ name: "DOMAIN", value: "platform.pennlabs.org" }], 52 | }, 53 | domains: undefined, 54 | djangoSettingsModule: "Platform.settings.production", 55 | }, 56 | }; 57 | 58 | test("Django Application -- Default", () => chartTest(buildDjangoChartDefault)); 59 | test("Django Application -- Example", () => chartTest(buildDjangoChartExample)); 60 | test("Django Application -- Duplicate Env", () => 61 | chartTest(buildDjangoChartDuplicateEnv)); 62 | test("Django Application -- Undefined Domains Chart", () => 63 | chartTest(buildDjangoIngressUndefinedDomainsChart)); 64 | 65 | function buildDjangoChartDefault(scope: Construct) { 66 | new DjangoApplication(scope, "platform", djangoTestConfig.default); 67 | } 68 | function buildDjangoChartExample(scope: Construct) { 69 | new DjangoApplication(scope, "platform", djangoTestConfig.example); 70 | } 71 | function buildDjangoIngressUndefinedDomainsChart(scope: Construct) { 72 | new DjangoApplication(scope, "platform", djangoTestConfig.undefinedDomains); 73 | } 74 | function buildDjangoChartDuplicateEnv(scope: Construct) { 75 | new DjangoApplication(scope, "platform", djangoTestConfig.duplicateEnv); 76 | } 77 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/application/react.test.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { ReactApplication } from "../../src/application"; 3 | import { chartTest } from "../utils"; 4 | 5 | /** 6 | * Test Configuration for ReactApplication 7 | * 8 | * default - assumes mostly default values for the configuration 9 | * example - sample parameters 10 | * duplicateEnv - env is defined twice, should not throw an error 11 | */ 12 | const reactTestConfig = { 13 | default: { 14 | deployment: { 15 | image: "pennlabs/penn-clubs-frontend", 16 | }, 17 | domain: { host: "pennclubs.com", paths: ["/"] }, 18 | ingressPaths: ["/"], 19 | }, 20 | example: { 21 | deployment: { 22 | image: "pennlabs/penn-clubs-frontend", 23 | replicas: 2, 24 | env: [{ name: "SOME_ENV", value: "environment variables are cool" }], 25 | }, 26 | domain: { host: "pennclubs.com", paths: ["/"] }, 27 | port: 8080, 28 | }, 29 | duplicateEnv: { 30 | deployment: { 31 | image: "pennlabs/penn-clubs-frontend", 32 | replicas: 2, 33 | env: [{ name: "DOMAIN", value: "pennclubs.com" }], 34 | }, 35 | domain: { host: "pennclubs.com", paths: ["/"] }, 36 | port: 80, 37 | }, 38 | }; 39 | 40 | test("React Application -- Default", () => chartTest(buildReactChartDefault)); 41 | test("React Application -- Example", () => chartTest(buildReactChartExample)); 42 | test("React Application -- Duplicate Env", () => 43 | chartTest(buildReactChartDuplicateEnv)); 44 | 45 | function buildReactChartDefault(scope: Construct) { 46 | new ReactApplication(scope, "react", reactTestConfig.default); 47 | } 48 | function buildReactChartExample(scope: Construct) { 49 | new ReactApplication(scope, "react", reactTestConfig.example); 50 | } 51 | function buildReactChartDuplicateEnv(scope: Construct) { 52 | new ReactApplication(scope, "react", reactTestConfig.duplicateEnv); 53 | } 54 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/application/redis.test.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { RedisApplication } from "../../src/application"; 3 | import { chartTest, failingTestNoAWSAccountId } from "../utils"; 4 | 5 | /** 6 | * Test Configuration for RedisApplication 7 | * 8 | * default - assumes the default values for the configuration 9 | * example - uses the customized values for the configuration 10 | */ 11 | const redisTestConfig = { 12 | default: {}, 13 | example: { 14 | deployment: { 15 | image: "custom-redis-image", 16 | tag: "5.0", 17 | }, 18 | port: 6380, 19 | createServiceAccount: true, 20 | }, 21 | persistent: { 22 | persistData: true, 23 | }, 24 | persistentWithCustomVolumes: { 25 | persistData: true, 26 | deployment: { 27 | secretMounts: [ 28 | { 29 | name: "example-mount-secret", 30 | mountPath: "/etc/redis", 31 | }, 32 | ], 33 | volumeMounts: [ 34 | { 35 | name: "example-mount", 36 | mountPath: "/etc/volumes", 37 | }, 38 | ], 39 | }, 40 | }, 41 | customConfigMap: { 42 | redisConfigMap: { 43 | name: "custom-config-map", 44 | config: "custom-config", 45 | }, 46 | }, 47 | }; 48 | 49 | function buildRedisChartDefault(scope: Construct) { 50 | new RedisApplication(scope, "redis", redisTestConfig.default); 51 | } 52 | function buildRedisChartExample(scope: Construct) { 53 | new RedisApplication(scope, "redis", redisTestConfig.example); 54 | } 55 | 56 | function buildRedisChartCustomConfigMap(scope: Construct) { 57 | new RedisApplication(scope, "redis", redisTestConfig.customConfigMap); 58 | } 59 | 60 | function buildRedisChartPersistent(scope: Construct) { 61 | new RedisApplication(scope, "redis", redisTestConfig.persistent); 62 | } 63 | function buildRedisChartPersistentWithCustomVolumes(scope: Construct) { 64 | new RedisApplication( 65 | scope, 66 | "redis", 67 | redisTestConfig.persistentWithCustomVolumes 68 | ); 69 | } 70 | 71 | test("Redis -- No ServiceAccount But CreateServiceAccount", () => 72 | failingTestNoAWSAccountId(buildRedisChartExample)); 73 | test("Redis Application -- Default", () => chartTest(buildRedisChartDefault)); 74 | test("Redis Application -- Example", () => chartTest(buildRedisChartExample)); 75 | test("Redis Application -- Persistence", () => 76 | chartTest(buildRedisChartPersistent)); 77 | test("Redis Application -- Custom ConfigMap", () => 78 | chartTest(buildRedisChartCustomConfigMap)); 79 | 80 | test("Redis Application -- Persistence with Custom Volumes", () => 81 | chartTest(buildRedisChartPersistentWithCustomVolumes)); 82 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/certificate.test.ts: -------------------------------------------------------------------------------- 1 | import { Chart, Testing } from "cdk8s"; 2 | import { Construct } from "constructs"; 3 | import { Certificate } from "../src"; 4 | import { chartTest, failingTest } from "./utils"; 5 | 6 | export function buildFailingCertificateChart(scope: Construct) { 7 | /** If domain is invalid certificate creation should throw an error. **/ 8 | new Certificate(scope, "serve", { host: "pennlabsorg", paths: ["/"] }); 9 | } 10 | 11 | export function buildCertificateChart(scope: Construct) { 12 | /** If domain is valid certificate creation should succeed. **/ 13 | new Certificate(scope, "serve", { host: "pennlabs.org", paths: ["/"] }); 14 | } 15 | 16 | /** Also, the Json Patch should not remove anything if not using PennLabsChart */ 17 | export const nonPennLabsChartTest = (build: (scope: Construct) => void) => { 18 | // Overriding env vars for testing purposes 19 | process.env.RELEASE_NAME = "RELEASE_NAME"; 20 | process.env.GIT_SHA = "TAG_FROM_CI"; 21 | process.env.AWS_ACCOUNT_ID = "TEST_AWS_ACCOUNT_ID"; 22 | 23 | const app = Testing.app(); 24 | const chart = new Chart(app, "kittyhawk"); 25 | build(chart); 26 | const results = Testing.synth(chart); 27 | expect(results).toMatchSnapshot(); 28 | }; 29 | 30 | test("Certificate -- Failing", () => failingTest(buildFailingCertificateChart)); 31 | test("Certificate -- Success", () => chartTest(buildCertificateChart)); 32 | test("Certificate (Non Penn Labs Chart) -- Success", () => 33 | nonPennLabsChartTest(buildCertificateChart)); 34 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/cronjob.test.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import cronTime from "cron-time-generator"; 3 | import { CronJob } from "../src/cronjob"; 4 | import { chartTest, failingTestNoGitSha } from "./utils"; 5 | 6 | export function buildCronjobVolumeChart(scope: Construct) { 7 | /** Tests a Cronjob with a volume. */ 8 | new CronJob(scope, "calculate-waits", { 9 | schedule: cronTime.every(5).minutes(), 10 | image: "pennlabs/penn-courses-backend", 11 | secret: "penn-courses", 12 | cmd: ["python", "manage.py", "calculatewaittimes"], 13 | secretMounts: [ 14 | { 15 | name: "labs-api-server", 16 | subPath: "ios-key", 17 | mountPath: "/app/ios_key.p8", 18 | }, 19 | ], 20 | }); 21 | } 22 | 23 | export function buildCronjobLimitsChart(scope: Construct) { 24 | /** Tests a Cronjob with success and failure limits. */ 25 | new CronJob(scope, "calculate-waits", { 26 | schedule: cronTime.every(5).minutes(), 27 | image: "pennlabs/penn-courses-backend", 28 | secret: "penn-courses", 29 | cmd: ["python", "manage.py", "calculatewaittimes"], 30 | successLimit: 3, 31 | failureLimit: 3, 32 | }); 33 | } 34 | 35 | export function buildCronjobWithServiceAccount(scope: Construct) { 36 | /** Tests a Cronjob with a service account. */ 37 | new CronJob(scope, "calculate-waits", { 38 | schedule: cronTime.every(5).minutes(), 39 | image: "pennlabs/penn-courses-backend", 40 | secret: "penn-courses", 41 | cmd: ["python", "manage.py", "calculatewaittimes"], 42 | createServiceAccount: true, 43 | }); 44 | } 45 | 46 | test("Cron Job with volume", () => chartTest(buildCronjobVolumeChart)); 47 | 48 | test("Cron Job with limits", () => chartTest(buildCronjobLimitsChart)); 49 | 50 | test("Cron Job -- No Git Sha", () => 51 | failingTestNoGitSha(buildCronjobVolumeChart)); 52 | 53 | test("Cron Job with service account", () => 54 | chartTest(buildCronjobWithServiceAccount)); 55 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/deployment.test.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { Container, Deployment } from "../src"; 3 | import { KubeServiceAccount } from "../src/imports/k8s"; 4 | import { chartTest, failingTestNoGitSha } from "./utils"; 5 | 6 | export function buildDeploymentDefault(scope: Construct) { 7 | new Deployment(scope, "container", { 8 | image: "pennlabs/website", 9 | tag: "latest", 10 | }); 11 | } 12 | 13 | export function buildDeploymentWithServiceAccount(scope: Construct) { 14 | new Deployment(scope, "container", { 15 | image: "pennlabs/website", 16 | tag: "latest", 17 | serviceAccount: new KubeServiceAccount(scope, "service-account", { 18 | metadata: { 19 | name: "service-account", 20 | }, 21 | }), 22 | }); 23 | } 24 | 25 | export function buildContainerDefault() { 26 | new Container({ 27 | image: "pennlabs/website", 28 | tag: "latest", 29 | }); 30 | } 31 | 32 | test("Deployment -- No Git Sha", () => 33 | failingTestNoGitSha(buildDeploymentDefault)); 34 | test("Deployment -- With Service Account", () => 35 | chartTest(buildDeploymentWithServiceAccount)); 36 | test("Deployment -- Default", () => chartTest(buildDeploymentDefault)); 37 | test("Container -- Default", () => chartTest(buildContainerDefault)); 38 | test("Container -- No Git Sha", () => 39 | failingContainerTestNoGitSha(buildContainerDefault)); 40 | 41 | export const failingContainerTestNoGitSha = (_: (scope: Construct) => void) => { 42 | const { GIT_SHA, ...env } = process.env; 43 | 44 | process.env = { 45 | ...env, 46 | RELEASE_NAME: "RELEASE_NAME", 47 | AWS_ACCOUNT_ID: "TEST_AWS_ACCOUNT_ID", 48 | }; 49 | 50 | expect(() => { 51 | buildContainerDefault(); 52 | }).toThrowError("process.exit: 1"); 53 | }; 54 | -------------------------------------------------------------------------------- /cdk/kittyhawk/test/utils.ts: -------------------------------------------------------------------------------- 1 | import { Testing } from "cdk8s"; 2 | import { Construct } from "constructs"; 3 | import { PennLabsChart } from "../src"; 4 | 5 | /** 6 | * Helper function to run each chart test 7 | * @param build function containing the constructs to be generated 8 | */ 9 | export const chartTest = (build: (scope: Construct) => void) => { 10 | // Overriding env vars for testing purposes 11 | process.env.RELEASE_NAME = "RELEASE_NAME"; 12 | process.env.GIT_SHA = "TAG_FROM_CI"; 13 | process.env.AWS_ACCOUNT_ID = "TEST_AWS_ACCOUNT_ID"; 14 | 15 | const app = Testing.app(); 16 | const chart = new PennLabsChart(app, "kittyhawk"); 17 | build(chart); 18 | const results = Testing.synth(chart); 19 | expect(results).toMatchSnapshot(); 20 | }; 21 | 22 | /** Helper function to run each chart test 23 | * @param build function containing the constructs to be generated 24 | */ 25 | export const failingTest = (build: (scope: Construct) => void) => { 26 | // Overriding env vars for testing purposes 27 | process.env.RELEASE_NAME = "RELEASE_NAME"; 28 | process.env.GIT_SHA = "TAG_FROM_CI"; 29 | process.env.AWS_ACCOUNT_ID = "TEST_AWS_ACCOUNT_ID"; 30 | 31 | const app = Testing.app(); 32 | expect(() => { 33 | const chart = new PennLabsChart(app, "kittyhawk"); 34 | build(chart); 35 | }).toThrowError(); 36 | }; 37 | 38 | /** 39 | * Helper function to replace process.exit with a function that throws an error for testing 40 | */ 41 | const mockExit = jest.spyOn(process, "exit").mockImplementation((code) => { 42 | throw new Error("process.exit: " + code); 43 | }); 44 | const mockConsoleError = jest.spyOn(console, "error").mockImplementation(() => { 45 | /* do nothing */ 46 | }); 47 | 48 | export const failingTestNoGitSha = (_: (scope: Construct) => void) => { 49 | const { GIT_SHA, ...env } = process.env; 50 | 51 | process.env = { 52 | ...env, 53 | RELEASE_NAME: "RELEASE_NAME", 54 | AWS_ACCOUNT_ID: "TEST_AWS_ACCOUNT_ID", 55 | }; 56 | 57 | expect(() => { 58 | const app = Testing.app(); 59 | new PennLabsChart(app, "kittyhawk"); 60 | }).toThrowError("process.exit: 1"); 61 | expect(mockConsoleError).toHaveBeenCalledTimes(1); 62 | expect(mockExit).toHaveBeenCalledWith(1); 63 | mockExit.mockClear(); 64 | }; 65 | 66 | export const failingTestNoAWSAccountId = ( 67 | build: (scope: Construct) => void 68 | ) => { 69 | process.env.RELEASE_NAME = "RELEASE_NAME"; 70 | process.env.GIT_SHA = "TAG_FROM_CI"; 71 | process.env.AWS_ACCOUNT_ID = ""; 72 | 73 | expect(() => { 74 | const app = Testing.app(); 75 | const chart = new PennLabsChart(app, "kittyhawk"); 76 | build(chart); 77 | }).toThrowError("process.exit: 1"); 78 | expect(mockConsoleError).toHaveBeenCalledTimes(1); 79 | expect(mockExit).toHaveBeenCalledWith(1); 80 | mockConsoleError.mockClear(); 81 | }; 82 | -------------------------------------------------------------------------------- /cdk/kittyhawk/tsconfig.dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "alwaysStrict": true, 4 | "declaration": true, 5 | "esModuleInterop": true, 6 | "experimentalDecorators": true, 7 | "inlineSourceMap": true, 8 | "inlineSources": true, 9 | "lib": [ 10 | "es2019" 11 | ], 12 | "module": "CommonJS", 13 | "noEmitOnError": false, 14 | "noFallthroughCasesInSwitch": true, 15 | "noImplicitAny": true, 16 | "noImplicitReturns": true, 17 | "noImplicitThis": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "resolveJsonModule": true, 21 | "strict": true, 22 | "strictNullChecks": true, 23 | "strictPropertyInitialization": true, 24 | "stripInternal": true, 25 | "target": "ES2019" 26 | }, 27 | "include": [ 28 | ".projenrc.js", 29 | "src/**/*.ts", 30 | "test/**/*.ts" 31 | ], 32 | "exclude": [ 33 | "node_modules" 34 | ], 35 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"npx projen\"." 36 | } 37 | -------------------------------------------------------------------------------- /cdk/kittyhawk/tsconfig.eslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "alwaysStrict": true, 4 | "declaration": true, 5 | "experimentalDecorators": true, 6 | "inlineSourceMap": true, 7 | "inlineSources": true, 8 | "lib": [ 9 | "es2018" 10 | ], 11 | "module": "CommonJS", 12 | "noEmitOnError": false, 13 | "noFallthroughCasesInSwitch": true, 14 | "noImplicitAny": true, 15 | "noImplicitReturns": true, 16 | "noImplicitThis": true, 17 | "noUnusedLocals": true, 18 | "noUnusedParameters": true, 19 | "resolveJsonModule": true, 20 | "strict": true, 21 | "strictNullChecks": true, 22 | "strictPropertyInitialization": true, 23 | "stripInternal": true, 24 | "target": "ES2018" 25 | }, 26 | "include": [ 27 | ".projenrc.js", 28 | "src/**/*.ts", 29 | "test/**/*.ts" 30 | ], 31 | "exclude": [ 32 | "node_modules" 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /cdk/kittyhawk/tsconfig.jest.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "alwaysStrict": true, 4 | "declaration": true, 5 | "experimentalDecorators": true, 6 | "inlineSourceMap": true, 7 | "inlineSources": true, 8 | "lib": [ 9 | "es2018" 10 | ], 11 | "module": "CommonJS", 12 | "noEmitOnError": false, 13 | "noFallthroughCasesInSwitch": true, 14 | "noImplicitAny": true, 15 | "noImplicitReturns": true, 16 | "noImplicitThis": true, 17 | "noUnusedLocals": true, 18 | "noUnusedParameters": true, 19 | "resolveJsonModule": true, 20 | "strict": true, 21 | "strictNullChecks": true, 22 | "strictPropertyInitialization": true, 23 | "stripInternal": true, 24 | "target": "ES2018", 25 | "esModuleInterop": true 26 | }, 27 | "include": [ 28 | ".projenrc.js", 29 | "src/**/*.ts", 30 | "test/**/*.ts" 31 | ], 32 | "exclude": [ 33 | "node_modules" 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /cdk/kittyhawk/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "rootDir": "src", 4 | "outDir": "lib", 5 | "alwaysStrict": true, 6 | "declaration": true, 7 | "esModuleInterop": true, 8 | "experimentalDecorators": true, 9 | "inlineSourceMap": true, 10 | "inlineSources": true, 11 | "lib": [ 12 | "es2019" 13 | ], 14 | "module": "CommonJS", 15 | "noEmitOnError": false, 16 | "noFallthroughCasesInSwitch": true, 17 | "noImplicitAny": true, 18 | "noImplicitReturns": true, 19 | "noImplicitThis": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "resolveJsonModule": true, 23 | "strict": true, 24 | "strictNullChecks": true, 25 | "strictPropertyInitialization": true, 26 | "stripInternal": true, 27 | "target": "ES2019" 28 | }, 29 | "include": [ 30 | "src/**/*.ts" 31 | ], 32 | "exclude": [], 33 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"npx projen\"." 34 | } 35 | -------------------------------------------------------------------------------- /cdk/kraken/.gitattributes: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "npx projen". 2 | 3 | *.snap linguist-generated 4 | /.eslintrc.json linguist-generated 5 | /.gitattributes linguist-generated 6 | /.gitignore linguist-generated 7 | /.npmignore linguist-generated 8 | /.prettierignore linguist-generated 9 | /.prettierrc.json linguist-generated 10 | /.projen/** linguist-generated 11 | /.projen/deps.json linguist-generated 12 | /.projen/files.json linguist-generated 13 | /.projen/tasks.json linguist-generated 14 | /LICENSE linguist-generated 15 | /package.json linguist-generated 16 | /tsconfig.dev.json linguist-generated 17 | /tsconfig.json linguist-generated 18 | /yarn.lock linguist-generated -------------------------------------------------------------------------------- /cdk/kraken/.gitignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "npx projen". 2 | !/.gitattributes 3 | !/.projen/tasks.json 4 | !/.projen/deps.json 5 | !/.projen/files.json 6 | !/package.json 7 | !/LICENSE 8 | !/.npmignore 9 | logs 10 | *.log 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | lerna-debug.log* 15 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | lib-cov 21 | coverage 22 | *.lcov 23 | .nyc_output 24 | build/Release 25 | node_modules/ 26 | jspm_packages/ 27 | *.tsbuildinfo 28 | .eslintcache 29 | *.tgz 30 | .yarn-integrity 31 | .cache 32 | /docs 33 | !/.projenrc.js 34 | /test-reports/ 35 | junit.xml 36 | /coverage/ 37 | !/.prettierignore 38 | !/.prettierrc.json 39 | !/test/ 40 | !/tsconfig.json 41 | !/tsconfig.dev.json 42 | !/src/ 43 | /lib 44 | /dist/ 45 | !/.eslintrc.json 46 | -------------------------------------------------------------------------------- /cdk/kraken/.npmignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "npx projen". 2 | /.projen/ 3 | /test-reports/ 4 | junit.xml 5 | /coverage/ 6 | /test/ 7 | /tsconfig.dev.json 8 | /src/ 9 | !/lib/ 10 | !/lib/**/*.js 11 | !/lib/**/*.d.ts 12 | dist 13 | /tsconfig.json 14 | /.github/ 15 | /.vscode/ 16 | /.idea/ 17 | /.projenrc.js 18 | tsconfig.tsbuildinfo 19 | /.eslintrc.json 20 | -------------------------------------------------------------------------------- /cdk/kraken/.prettierignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "npx projen". 2 | src/imports 3 | -------------------------------------------------------------------------------- /cdk/kraken/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "overrides": [] 3 | } 4 | -------------------------------------------------------------------------------- /cdk/kraken/.projen/deps.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": [ 3 | { 4 | "name": "@types/jest", 5 | "type": "build" 6 | }, 7 | { 8 | "name": "@types/node", 9 | "version": "^12", 10 | "type": "build" 11 | }, 12 | { 13 | "name": "@typescript-eslint/eslint-plugin", 14 | "version": "^5", 15 | "type": "build" 16 | }, 17 | { 18 | "name": "@typescript-eslint/parser", 19 | "version": "^5", 20 | "type": "build" 21 | }, 22 | { 23 | "name": "codecov", 24 | "type": "build" 25 | }, 26 | { 27 | "name": "eslint-config-prettier", 28 | "type": "build" 29 | }, 30 | { 31 | "name": "eslint-import-resolver-node", 32 | "type": "build" 33 | }, 34 | { 35 | "name": "eslint-import-resolver-typescript", 36 | "type": "build" 37 | }, 38 | { 39 | "name": "eslint-plugin-import", 40 | "type": "build" 41 | }, 42 | { 43 | "name": "eslint-plugin-prettier", 44 | "type": "build" 45 | }, 46 | { 47 | "name": "eslint", 48 | "version": "^8", 49 | "type": "build" 50 | }, 51 | { 52 | "name": "jest", 53 | "type": "build" 54 | }, 55 | { 56 | "name": "jest-junit", 57 | "version": "^13", 58 | "type": "build" 59 | }, 60 | { 61 | "name": "json-schema", 62 | "type": "build" 63 | }, 64 | { 65 | "name": "prettier", 66 | "type": "build" 67 | }, 68 | { 69 | "name": "projen", 70 | "type": "build" 71 | }, 72 | { 73 | "name": "ts-jest", 74 | "type": "build" 75 | }, 76 | { 77 | "name": "typedoc", 78 | "version": "^0.21.4", 79 | "type": "build" 80 | }, 81 | { 82 | "name": "typescript", 83 | "type": "build" 84 | }, 85 | { 86 | "name": "cdkactions", 87 | "type": "runtime" 88 | }, 89 | { 90 | "name": "constructs", 91 | "type": "runtime" 92 | }, 93 | { 94 | "name": "ts-dedent", 95 | "type": "runtime" 96 | } 97 | ], 98 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"npx projen\"." 99 | } 100 | -------------------------------------------------------------------------------- /cdk/kraken/.projen/files.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [ 3 | ".eslintrc.json", 4 | ".gitattributes", 5 | ".gitignore", 6 | ".npmignore", 7 | ".prettierignore", 8 | ".prettierrc.json", 9 | ".projen/deps.json", 10 | ".projen/files.json", 11 | ".projen/tasks.json", 12 | "LICENSE", 13 | "tsconfig.dev.json", 14 | "tsconfig.json" 15 | ], 16 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"npx projen\"." 17 | } 18 | -------------------------------------------------------------------------------- /cdk/kraken/.projenrc.js: -------------------------------------------------------------------------------- 1 | const { TypeScriptProject } = require('projen/lib/typescript'); 2 | const common = require('../projen-common'); 3 | 4 | const project = new TypeScriptProject({ 5 | name: '@pennlabs/kraken', 6 | description: 'cdkactions construct abstractions built for Penn Labs', 7 | deps: ['cdkactions', 'constructs', 'ts-dedent'], 8 | devDeps: ['codecov'], 9 | keywords: ['cdk', 'github', 'actions', 'constructs', 'cdkactions'], 10 | homepage: 'https://kraken.pennlabs.org', 11 | repositoryDirectory: 'cdk/kraken', 12 | ...common.options, 13 | }); 14 | 15 | project.addFields({['version']: '0.8.12'}); 16 | project.prettier?.ignoreFile?.addPatterns('src/imports'); 17 | project.testTask.steps.forEach(step => { 18 | if (step.exec) { 19 | step.exec = step.exec.replace(' --updateSnapshot', ''); 20 | } 21 | }); 22 | project.synth(); 23 | -------------------------------------------------------------------------------- /cdk/kraken/.versionrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "packageFiles": [ 3 | { 4 | "filename": "version.json", 5 | "type": "json" 6 | } 7 | ], 8 | "bumpFiles": [ 9 | { 10 | "filename": "version.json", 11 | "type": "json" 12 | } 13 | ], 14 | "commitAll": true, 15 | "scripts": { 16 | "postbump": "npx projen && git add ." 17 | } 18 | } -------------------------------------------------------------------------------- /cdk/kraken/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## X.Y.Z (UNRELEASED) 4 | 5 | ## 0.8.12 (2023-10-03) 6 | 7 | - Update DjangoCheckJob to have codecov token 8 | 9 | ## 0.8.11 (2023-10-02) 10 | 11 | - Update DjangoCheckJob to account for codecov deprecation 12 | 13 | ## 0.8.10 (2022-11-05) 14 | 15 | - Docker image: support build args 16 | 17 | ## 0.8.6 (2022-04-08) 18 | 19 | - Upgrade dependencies and fix projen version configuration 20 | 21 | ## 0.8.5 (2022-04-05) 22 | 23 | - CDK publish: fix conditional publish check 24 | 25 | ## 0.8.4 (2022-04-03) 26 | 27 | - CDK publish: fix version_check id location 28 | 29 | ## 0.8.3 (2022-04-03) 30 | 31 | - CDK publish: force jq exit code to be 0 32 | 33 | ## 0.8.2 (2022-04-03) 34 | 35 | - Fix CDK publish npm version check output 36 | 37 | ## 0.8.1 (2022-04-01) 38 | 39 | - Fix deploy job environment variable handling 40 | - Fix CDK publish stack by installing jq 41 | 42 | ## 0.8.0 (2022-03-29) 43 | 44 | - Migrate to kittyhawk for deployments 45 | - Modify CDK stack to only publish when a new version is pushed 46 | 47 | ## 0.7.1 (2021-12-16) 48 | 49 | - Modify PyPI to correctly publish (again) 50 | 51 | ## 0.7.0 (2021-12-04) 52 | 53 | - Modify PyPI python versions to be strings 54 | - Modify PyPI to correctly publish 55 | - Migrate to ts-dedent 56 | 57 | ## 0.6.4 (2021-10-04) 58 | 59 | - Modify PyPI publish to use poetry 60 | 61 | ## 0.6.3 (2021-08-26) 62 | 63 | - Pin OS to buster within django check 64 | 65 | ## 0.6.2 (2021-04-04) 66 | 67 | - Add poetry install to PyPI stack 68 | 69 | ## 0.6.1 (2021-04-04) 70 | 71 | - Fix PyPI job dependencies 72 | 73 | ## 0.6.0 (2021-04-01) 74 | 75 | - Modify PyPI stack to use a matrix 76 | 77 | ## 0.5.1 (2021-03-17) 78 | 79 | - Remove unused env var from deploy job 80 | 81 | ## 0.5.0 (2021-02-25) 82 | 83 | - Modify deploy job to deploy to AWS EKS 84 | 85 | ## 0.4.12 (2021-02-17) 86 | 87 | - Hotfix deploy job part 2 88 | 89 | ## 0.4.11 (2021-02-17) 90 | 91 | - Hotfix deploy job 92 | 93 | ## 0.4.10 (2021-02-13) 94 | 95 | - Modify deploy script to fully fail if a single command fails 96 | 97 | ## 0.4.9 (2021-02-11) 98 | 99 | - Create an auto-approve stack for dependabot PRs 100 | 101 | ## 0.4.8 (2021-02-04) 102 | 103 | - Only publish docs in cdk stack on default branch 104 | 105 | ## 0.4.7 (2021-01-30) 106 | 107 | - Bugfix post-integration publish job again 108 | 109 | ## 0.4.6 (2021-01-30) 110 | 111 | - Bugfix post-integration publish job 112 | 113 | ## 0.4.5 (2021-01-30) 114 | 115 | - Update `docker/build-push-action` to v2 116 | - Complete integration test job 117 | 118 | ## 0.3.10 (2020-12-30) 119 | 120 | - Initial beta release 121 | -------------------------------------------------------------------------------- /cdk/kraken/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Penn Labs 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /cdk/kraken/README.md: -------------------------------------------------------------------------------- 1 | # kraken 2 | 3 | A cdk library for [cdkactions](https://github.com/ArmaanT/cdkactions/). 4 | 5 | ## Documentation 6 | 7 | See [Kraken documentation](https://kraken.pennlabs.org/). 8 | 9 | ## Getting started 10 | 11 | When creating a new project, do the following: 12 | 13 | * Follow the [cdkactions getting started guide](https://github.com/ArmaanT/cdkactions/blob/master/docs/getting-started/typescript.md) to initialize the cdkactions project. 14 | * Run `yarn install @pennlabs/kraken` 15 | * Configure cdkactions (See the following subjections) 16 | * Finally, run `yarn build` within the `.github/cdk` folder and commit your changes. 17 | 18 | ### Django + React Project 19 | 20 | Most Penn Labs products follow the mold of a Django project in the `backend` directory and a React project in the `frontend` directory. 21 | 22 | Since this is a common use-case, kraken provides an `LabsApplicationStack` that can easily configure the CI for this type of project. 23 | 24 | To use an `LabsApplicationStack` stack, just replace the generated stack with: 25 | 26 | ``` javascript 27 | new LabsApplicationStack(app, { 28 | djangoProjectName: 'exampleDjangoProject', 29 | dockerImageBaseName: 'example-product', 30 | }); 31 | ``` 32 | 33 | This configuration will lint, test, build, publish docker images for, and deploy both the Django project and a React project. The published docker images will be named: `$dockerImageBaseName-backend` and `$dockerImageBaseName-frontend` 34 | 35 | ### Different Configuration 36 | 37 | If your repo is different from the normal 1 Django and 1 React project per repo, you can utilize the `DjangoProject` and `ReactProject` classes. For example, if you had a single Django project with multiple frontends, you could add something like the following inside of the generated Stack: 38 | 39 | ``` javascript 40 | const workflow = new Workflow(this, 'workflow', { 41 | name: 'Workflow', 42 | on: 'push', 43 | }); 44 | 45 | const django = new DjangoProject(workflow, 46 | { 47 | projectName: 'djangoProject', 48 | path: 'backend', 49 | imageName: 'project-backend', 50 | }); 51 | 52 | const reactOne = new ReactProject(workflow, 53 | { 54 | id: 'one', 55 | path: 'frontendOne', 56 | imageName: 'project-frontendOne', 57 | }); 58 | 59 | const reactTwo = new ReactProject(workflow, 60 | { 61 | id: 'two', 62 | path: 'frontendTwo', 63 | imageName: 'project-frontendTwo', 64 | }); 65 | 66 | new DeployJob(workflow, {}, 67 | { 68 | needs: [django.publishJobId, reactOne.publishJobId, reactTwo.publishJobId], 69 | }); 70 | ``` 71 | 72 | This configuration will lint, test, build, publish docker images for the Django project as well as the two React projects and then finally deploy the application. 73 | -------------------------------------------------------------------------------- /cdk/kraken/src/auto-approve.ts: -------------------------------------------------------------------------------- 1 | import { Workflow, Stack, WorkflowProps, Job } from "cdkactions"; 2 | import { Construct } from "constructs"; 3 | 4 | /** 5 | * Auto-approve PRs opened by dependabot. Used to fulfil requirements 6 | * needed to automatically merge using mergify 7 | */ 8 | export class AutoApproveStack extends Stack { 9 | /** 10 | * 11 | * @param scope cdkactions App instance. 12 | * @param overrides Optional overrides for the stack. 13 | */ 14 | public constructor(scope: Construct, overrides?: Partial) { 15 | super(scope, "autoapprove"); 16 | const workflow = new Workflow(this, "autoapprove", { 17 | name: "Auto Approve dependabot PRs", 18 | on: "pullRequest", 19 | ...overrides, 20 | }); 21 | 22 | new Job(workflow, "approve", { 23 | runsOn: "ubuntu-latest", 24 | steps: [ 25 | { 26 | uses: "hmarr/auto-approve-action@v2.0.0", 27 | if: "github.actor == 'dependabot[bot]'", 28 | with: { 29 | "github-token": "${{ secrets.BOT_GITHUB_PAT }}", 30 | }, 31 | }, 32 | ], 33 | }); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /cdk/kraken/src/deploy.ts: -------------------------------------------------------------------------------- 1 | import { CheckoutJob, Workflow, JobProps } from "cdkactions"; 2 | import dedent from "ts-dedent"; 3 | 4 | /** 5 | * Optional props to configure the deploy job. 6 | */ 7 | export interface DeployJobProps { 8 | /** 9 | * Deploy tag to set in kittyhawk. 10 | * @default current git sha 11 | */ 12 | deployTag?: string; 13 | 14 | /** 15 | * Branch to limit deploys to. 16 | * @default master 17 | */ 18 | defaultBranch?: string; 19 | } 20 | 21 | /** 22 | * A job to deploy an application using Kittyhawk. 23 | */ 24 | export class DeployJob extends CheckoutJob { 25 | /** 26 | * 27 | * @param scope cdkactions Workflow instance 28 | * @param config Optional configuration for the deploy job. 29 | * @param overrides Optional overrides for the job. 30 | */ 31 | public constructor( 32 | scope: Workflow, 33 | config?: DeployJobProps, 34 | overrides?: Partial 35 | ) { 36 | // Build config 37 | const fullConfig: Required = { 38 | deployTag: "${{ github.sha }}", 39 | defaultBranch: "master", 40 | ...config, 41 | }; 42 | 43 | super(scope, "deploy", { 44 | runsOn: "ubuntu-latest", 45 | if: `github.ref == 'refs/heads/${fullConfig.defaultBranch}'`, 46 | steps: [ 47 | { 48 | id: "synth", 49 | name: "Synth cdk8s manifests", 50 | run: dedent`cd k8s 51 | yarn install --frozen-lockfile 52 | 53 | # get repo name (by removing owner/organization) 54 | export RELEASE_NAME=\${REPOSITORY#*/} 55 | 56 | # Export RELEASE_NAME as an output 57 | echo "::set-output name=RELEASE_NAME::$RELEASE_NAME" 58 | 59 | yarn build`, 60 | env: { 61 | GIT_SHA: fullConfig.deployTag, 62 | REPOSITORY: "${{ github.repository }}", 63 | AWS_ACCOUNT_ID: "${{ secrets.AWS_ACCOUNT_ID }}", 64 | }, 65 | }, 66 | { 67 | name: "Deploy", 68 | run: dedent`aws eks --region us-east-1 update-kubeconfig --name production --role-arn arn:aws:iam::\${AWS_ACCOUNT_ID}:role/kubectl 69 | 70 | # get repo name from synth step 71 | RELEASE_NAME=\${{ steps.synth.outputs.RELEASE_NAME }} 72 | 73 | # Deploy 74 | kubectl apply -f k8s/dist/ -l app.kubernetes.io/component=certificate 75 | kubectl apply -f k8s/dist/ --prune -l app.kubernetes.io/part-of=$RELEASE_NAME`, 76 | env: { 77 | AWS_ACCOUNT_ID: "${{ secrets.AWS_ACCOUNT_ID }}", 78 | AWS_ACCESS_KEY_ID: "${{ secrets.GH_AWS_ACCESS_KEY_ID }}", 79 | AWS_SECRET_ACCESS_KEY: "${{ secrets.GH_AWS_SECRET_ACCESS_KEY }}", 80 | }, 81 | }, 82 | ], 83 | ...overrides, 84 | }); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /cdk/kraken/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./auto-approve"; 2 | export * from "./cdk"; 3 | export * from "./deploy"; 4 | export * from "./django-project"; 5 | export * from "./django"; 6 | export * from "./docker"; 7 | export * from "./integration-tests"; 8 | export * from "./labs-application"; 9 | export * from "./postintegrationimagepublishjob"; 10 | export * from "./pypi"; 11 | export * from "./react-project"; 12 | export * from "./react"; 13 | export * from "./utils"; 14 | -------------------------------------------------------------------------------- /cdk/kraken/src/postintegrationimagepublishjob.ts: -------------------------------------------------------------------------------- 1 | import { CheckoutJob, Workflow, JobProps } from "cdkactions"; 2 | 3 | /** 4 | * Props to configure the post integration test docker image publish job. 5 | */ 6 | export interface PostIntegrationPublishJobProps { 7 | /** 8 | * Branch to restrict publishing to 9 | * @default master 10 | */ 11 | defaultBranch?: string; 12 | 13 | /** 14 | * A list of job IDs that built docker images. 15 | */ 16 | dockerBuildIds: string[]; 17 | 18 | /** 19 | * A list of the docker images to publish. 20 | */ 21 | dockerImages: string[]; 22 | 23 | /** 24 | * Username to authenticate with Docker Hub. 25 | * @default load the "DOCKER_USERNAME" GitHub Actions secret. 26 | */ 27 | dockerUsername?: string; 28 | 29 | /** 30 | * Password to authenticate with Docker Hub. 31 | * @default load the "DOCKER_PASSWORD" GitHub Actions secret. 32 | */ 33 | dockerPassword?: string; 34 | } 35 | 36 | /** 37 | * A job to publish Docker images after completing an integration test. 38 | */ 39 | export class PostIntegrationPublishJob extends CheckoutJob { 40 | /** 41 | * 42 | * @param scope cdkactions Workflow instance. 43 | * @param id Id of the job. 44 | * @param config Configuration for the post integration tests publish job. 45 | * @param overrides Optional overrides for the job. 46 | */ 47 | public constructor( 48 | scope: Workflow, 49 | config: PostIntegrationPublishJobProps, 50 | overrides?: Partial 51 | ) { 52 | // Build config 53 | const fullConfig: Required = { 54 | defaultBranch: "master", 55 | dockerUsername: "${{ secrets.DOCKER_USERNAME }}", 56 | dockerPassword: "${{ secrets.DOCKER_PASSWORD }}", 57 | ...config, 58 | }; 59 | 60 | // Create job 61 | super(scope, "post-integration-publish", { 62 | name: "Publish Images", 63 | runsOn: "ubuntu-latest", 64 | if: `github.ref == 'refs/heads/${fullConfig.defaultBranch}'`, 65 | steps: [ 66 | { 67 | uses: "actions/download-artifact@v2", 68 | }, 69 | { 70 | uses: "geekyeggo/delete-artifact@v1", 71 | with: { 72 | name: fullConfig.dockerBuildIds.join("\n"), 73 | }, 74 | }, 75 | { 76 | name: "Load docker images", 77 | run: fullConfig.dockerBuildIds 78 | .map((id) => `docker load --input ${id}/image.tar`) 79 | .join("\n"), 80 | }, 81 | { 82 | uses: "docker/login-action@v3", 83 | with: { 84 | username: fullConfig.dockerUsername, 85 | password: fullConfig.dockerPassword, 86 | }, 87 | }, 88 | { 89 | name: "Push docker images", 90 | run: fullConfig.dockerImages 91 | .map((image) => `docker push -a ${image}`) 92 | .join("\n"), 93 | }, 94 | ], 95 | ...overrides, 96 | }); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /cdk/kraken/src/react.ts: -------------------------------------------------------------------------------- 1 | import { Workflow, JobProps, CheckoutJob } from "cdkactions"; 2 | import dedent from "ts-dedent"; 3 | import { buildId, buildName } from "./utils"; 4 | 5 | /** 6 | * Optional props to configure the React check job. 7 | */ 8 | export interface ReactCheckJobProps { 9 | /** 10 | * A custom id to append onto job name and ids. Useful when using 11 | * multiple instances of ReactCheckJob in a single workflow. 12 | * @default no suffix 13 | */ 14 | id?: string; 15 | 16 | /** 17 | * Node version to test the project with. 18 | * @default "14" 19 | */ 20 | nodeVersion?: string; 21 | 22 | /** 23 | * Location of the React project within the repo 24 | * @default "." 25 | */ 26 | path?: string; 27 | } 28 | 29 | /** 30 | * A job to lint and test a React project as well as upload code coverage. 31 | */ 32 | export class ReactCheckJob extends CheckoutJob { 33 | /** 34 | * 35 | * @param scope cdkactions Workflow instance. 36 | * @param config Optional configuration for the React check job. 37 | * @param overrides Optional overrides for the job. 38 | */ 39 | public constructor( 40 | scope: Workflow, 41 | config?: ReactCheckJobProps, 42 | overrides?: Partial 43 | ) { 44 | // Build config 45 | const fullConfig: Required = { 46 | id: "", 47 | nodeVersion: "14", 48 | path: ".", 49 | ...config, 50 | }; 51 | 52 | // Create Job 53 | super(scope, buildId("react-check", fullConfig.id), { 54 | name: buildName("React Check", fullConfig.id), 55 | runsOn: "ubuntu-latest", 56 | steps: [ 57 | { 58 | name: "Cache", 59 | uses: "actions/cache@v4", 60 | with: { 61 | path: "**/node_modules", 62 | key: `v0-\${{ hashFiles('${fullConfig.path}/yarn.lock') }}`, 63 | }, 64 | }, 65 | { 66 | name: "Install Dependencies", 67 | run: dedent`cd ${fullConfig.path} 68 | yarn install --frozen-lockfile`, 69 | }, 70 | { 71 | name: "Lint", 72 | run: dedent`cd ${fullConfig.path} 73 | yarn lint`, 74 | }, 75 | { 76 | name: "Test", 77 | run: dedent`cd ${fullConfig.path} 78 | yarn test`, 79 | }, 80 | { 81 | name: "Upload Code Coverage", 82 | run: dedent`ROOT=$(pwd) 83 | cd ${fullConfig.path} 84 | yarn run codecov -p $ROOT -F frontend`, 85 | }, 86 | ], 87 | container: { 88 | image: `node:${fullConfig.nodeVersion}`, 89 | }, 90 | ...overrides, 91 | }); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /cdk/kraken/src/utils.ts: -------------------------------------------------------------------------------- 1 | export const buildId = (id: string, suffix: string) => 2 | suffix ? `${id}-${suffix}` : id; 3 | export const buildName = (name: string, id: string) => `${name} ${id}`.trim(); 4 | -------------------------------------------------------------------------------- /cdk/kraken/test/__snapshots__/auto-approve.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`default 1`] = ` 4 | "# Generated by cdkactions. Do not modify 5 | # Generated as part of the 'autoapprove' stack. 6 | name: Auto Approve dependabot PRs 7 | on: pull_request 8 | jobs: 9 | approve: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: hmarr/auto-approve-action@v2.0.0 13 | if: github.actor == 'dependabot[bot]' 14 | with: 15 | github-token: \${{ secrets.BOT_GITHUB_PAT }} 16 | " 17 | `; 18 | -------------------------------------------------------------------------------- /cdk/kraken/test/__snapshots__/cdk.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`default 1`] = ` 4 | "# Generated by cdkactions. Do not modify 5 | # Generated as part of the 'example' stack. 6 | name: Publish example 7 | on: 8 | push: 9 | paths: 10 | - cdk/example/** 11 | jobs: 12 | publish: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Cache 17 | uses: actions/cache@v4 18 | with: 19 | path: \\"**/node_modules\\" 20 | key: v0-\${{ hashFiles('cdk/example/yarn.lock') }} 21 | - name: Install Dependencies 22 | run: |- 23 | cd cdk/example 24 | yarn install --frozen-lockfile 25 | - name: Test 26 | run: |- 27 | cd cdk/example 28 | yarn test 29 | - name: Upload Code Coverage 30 | run: |- 31 | ROOT=$(pwd) 32 | cd cdk/example 33 | yarn run codecov -p $ROOT -F example 34 | - name: Install jq 35 | run: |- 36 | curl -sSLo /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 37 | chmod +x /usr/bin/jq 38 | - name: Check if version is already published to npm 39 | id: version_check 40 | run: |- 41 | cd cdk/example 42 | PACKAGE=$(node -p \\"require('./package.json').name\\") 43 | VERSION=$(node -p \\"require('./package.json').version\\") 44 | NEW_VERSION=$(yarn info $PACKAGE versions --json | jq \\".data | any(. == \\\\\\"$VERSION\\\\\\") | not\\") 45 | echo \\"::set-output name=NEW_VERSION::$NEW_VERSION\\" 46 | - name: Publish to npm 47 | run: |- 48 | cd cdk/example 49 | yarn compile 50 | yarn package 51 | mv dist/js/*.tgz dist/js/example.tgz 52 | yarn publish --non-interactive --access public dist/js/example.tgz 53 | if: github.ref == 'refs/heads/master' && steps.version_check.outputs.NEW_VERSION == 'true' 54 | env: 55 | NPM_AUTH_TOKEN: \${{ secrets.NPM_AUTH_TOKEN }} 56 | - name: Build docs 57 | run: |- 58 | cd cdk/example 59 | yarn docgen 60 | - name: Publish docs 61 | if: github.ref == 'refs/heads/master' 62 | uses: peaceiris/actions-gh-pages@v3 63 | with: 64 | personal_token: \${{ secrets.BOT_GITHUB_PAT }} 65 | external_repository: pennlabs/example-docs 66 | cname: example.pennlabs.org 67 | publish_branch: master 68 | publish_dir: cdk/example/docs 69 | user_name: github-actions 70 | user_email: github-actions[bot]@users.noreply.github.com 71 | container: 72 | image: node:14 73 | " 74 | `; 75 | -------------------------------------------------------------------------------- /cdk/kraken/test/__snapshots__/pypi.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`default 1`] = ` 4 | "# Generated by cdkactions. Do not modify 5 | # Generated as part of the 'pypi' stack. 6 | name: Build and Publish 7 | on: 8 | push: 9 | branches: 10 | - \\"**\\" 11 | tags: 12 | - \\"[0-9]+.[0-9]+.[0-9]+\\" 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | python-version: 19 | - \\"3.7\\" 20 | - \\"3.8\\" 21 | - \\"3.9\\" 22 | - \\"3.10\\" 23 | steps: 24 | - uses: actions/checkout@v4 25 | - name: Set up Python \${{ matrix.python-version }} 26 | uses: actions/setup-python@v2 27 | with: 28 | python-version: \${{ matrix.python-version }} 29 | - name: Install dependencies 30 | run: pip install poetry tox tox-gh-actions codecov 31 | - name: Test 32 | run: tox 33 | - name: Upload Code Coverage 34 | run: codecov 35 | publish: 36 | runs-on: ubuntu-latest 37 | container: 38 | image: python:3.8 39 | needs: test 40 | if: startsWith(github.ref, 'refs/tags') 41 | steps: 42 | - uses: actions/checkout@v4 43 | - name: Install dependencies 44 | run: pip install poetry 45 | - name: Verify tag 46 | shell: bash 47 | run: |- 48 | GIT_TAG=\${GITHUB_REF/refs\\\\/tags\\\\//} 49 | LIBRARY_VERSION=$(poetry version -s) 50 | if [[ \\"$GIT_TAG\\" != \\"$LIBRARY_VERSION\\" ]]; then echo \\"Tag ($GIT_TAG) does not match poetry version ($LIBRARY_VERSION)\\"; exit 1; fi 51 | - name: Build 52 | run: poetry build 53 | - name: Publish 54 | run: poetry publish 55 | env: 56 | POETRY_PYPI_TOKEN_PYPI: \${{ secrets.PYPI_PASSWORD }} 57 | " 58 | `; 59 | -------------------------------------------------------------------------------- /cdk/kraken/test/auto-approve.test.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | import { AutoApproveStack } from "../src"; 3 | import { TestingApp } from "./utils"; 4 | 5 | test("default", () => { 6 | const app = TestingApp({ createValidateWorkflow: false }); 7 | new AutoApproveStack(app); 8 | app.synth(); 9 | expect(fs.readdirSync(app.outdir)).toEqual(["cdkactions_autoapprove.yaml"]); 10 | expect( 11 | fs.readFileSync(`${app.outdir}/cdkactions_autoapprove.yaml`, "utf-8") 12 | ).toMatchSnapshot(); 13 | }); 14 | -------------------------------------------------------------------------------- /cdk/kraken/test/cdk.test.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | import { CDKPublishStack } from "../src"; 3 | import { TestingApp } from "./utils"; 4 | 5 | test("default", () => { 6 | const app = TestingApp({ createValidateWorkflow: false }); 7 | new CDKPublishStack(app, "example"); 8 | app.synth(); 9 | expect(fs.readdirSync(app.outdir)).toEqual(["cdkactions_example.yaml"]); 10 | expect( 11 | fs.readFileSync(`${app.outdir}/cdkactions_example.yaml`, "utf-8") 12 | ).toMatchSnapshot(); 13 | }); 14 | -------------------------------------------------------------------------------- /cdk/kraken/test/custom.test.ts: -------------------------------------------------------------------------------- 1 | import { Workflow } from "cdkactions"; 2 | import { DeployJob, DjangoProject, ReactProject } from "../src"; 3 | 4 | test("workflow with 2 django+react projects", () => { 5 | const workflow = new Workflow(undefined as any, "workflow", { 6 | name: "Workflow", 7 | on: "push", 8 | }); 9 | 10 | const djangoOne = new DjangoProject(workflow, { 11 | id: "one", 12 | projectName: "projectOne", 13 | path: "backendOne", 14 | imageName: "imageOne-backend", 15 | }); 16 | const djangoTwo = new DjangoProject(workflow, { 17 | id: "two", 18 | projectName: "projectTwo", 19 | path: "backendTwo", 20 | imageName: "imageTwo-backend", 21 | }); 22 | 23 | const reactOne = new ReactProject(workflow, { 24 | id: "one", 25 | path: "frontendOne", 26 | imageName: "imageOne-frontend", 27 | }); 28 | const reactTwo = new ReactProject(workflow, { 29 | id: "two", 30 | path: "frontendTwo", 31 | imageName: "imageTwo-frontend", 32 | }); 33 | new DeployJob( 34 | workflow, 35 | {}, 36 | { 37 | needs: [ 38 | djangoOne.publishJobId, 39 | djangoTwo.publishJobId, 40 | reactOne.publishJobId, 41 | reactTwo.publishJobId, 42 | ], 43 | } 44 | ); 45 | 46 | expect(workflow.toGHAction()).toMatchSnapshot(); 47 | }); 48 | -------------------------------------------------------------------------------- /cdk/kraken/test/django-project.test.ts: -------------------------------------------------------------------------------- 1 | import { Workflow } from "cdkactions"; 2 | import { DjangoProject } from "../src"; 3 | 4 | test("default", () => { 5 | const workflow = new Workflow(undefined as any, "workflow", { 6 | name: "Workflow", 7 | on: "push", 8 | }); 9 | new DjangoProject(workflow, { 10 | projectName: "example", 11 | imageName: "example", 12 | }); 13 | expect(workflow.toGHAction()).toMatchSnapshot(); 14 | }); 15 | 16 | test("custom id", () => { 17 | const workflow = new Workflow(undefined as any, "workflow", { 18 | name: "Workflow", 19 | on: "push", 20 | }); 21 | new DjangoProject(workflow, { 22 | projectName: "example", 23 | imageName: "example", 24 | id: "custom", 25 | }); 26 | expect(workflow.toGHAction()).toMatchSnapshot(); 27 | }); 28 | -------------------------------------------------------------------------------- /cdk/kraken/test/django.test.ts: -------------------------------------------------------------------------------- 1 | import { DjangoCheckJob } from "../src"; 2 | 3 | test("default", () => { 4 | const dc = new DjangoCheckJob(undefined as any, { projectName: "example" }); 5 | expect(dc.toGHAction()).toMatchSnapshot(); 6 | }); 7 | 8 | test("different python version", () => { 9 | const dc = new DjangoCheckJob(undefined as any, { 10 | pythonVersion: "2.7", 11 | projectName: "example", 12 | }); 13 | expect(dc.toGHAction()).toMatchSnapshot(); 14 | }); 15 | 16 | test("different directory", () => { 17 | const dc = new DjangoCheckJob(undefined as any, { 18 | path: "backend", 19 | projectName: "example", 20 | }); 21 | expect(dc.toGHAction()).toMatchSnapshot(); 22 | }); 23 | 24 | test("no lint", () => { 25 | const dc = new DjangoCheckJob(undefined as any, { 26 | projectName: "example", 27 | black: false, 28 | flake8: false, 29 | }); 30 | expect(dc.toGHAction()).toMatchSnapshot(); 31 | }); 32 | 33 | test("with overrides", () => { 34 | const dc = new DjangoCheckJob( 35 | undefined as any, 36 | { projectName: "example" }, 37 | { continueOnError: true } 38 | ); 39 | expect(dc.toGHAction()).toMatchSnapshot(); 40 | }); 41 | -------------------------------------------------------------------------------- /cdk/kraken/test/docker.test.ts: -------------------------------------------------------------------------------- 1 | import { DockerPublishJob } from "../src"; 2 | 3 | // DockerPublish 4 | test("default", () => { 5 | const dc = new DockerPublishJob(undefined as any, "publish", { 6 | imageName: "example", 7 | }); 8 | expect(dc.toGHAction()).toMatchSnapshot(); 9 | }); 10 | 11 | test("disable cache", () => { 12 | const dc = new DockerPublishJob(undefined as any, "publish", { 13 | imageName: "example", 14 | cache: false, 15 | }); 16 | expect(dc.toGHAction()).toMatchSnapshot(); 17 | }); 18 | 19 | test("no publish", () => { 20 | const dc = new DockerPublishJob(undefined as any, "publish", { 21 | imageName: "example", 22 | noPublish: true, 23 | }); 24 | expect(dc.toGHAction()).toMatchSnapshot(); 25 | }); 26 | 27 | test("with build args", () => { 28 | const dc = new DockerPublishJob(undefined as any, "publish", { 29 | imageName: "example", 30 | buildArgs: { 31 | FOO: "bar", 32 | BAR: "baz", 33 | }, 34 | }); 35 | expect(dc.toGHAction()).toMatchSnapshot(); 36 | }); 37 | 38 | test("with overrides", () => { 39 | const dc = new DockerPublishJob( 40 | undefined as any, 41 | "publish", 42 | { imageName: "example" }, 43 | { continueOnError: true } 44 | ); 45 | expect(dc.toGHAction()).toMatchSnapshot(); 46 | }); 47 | -------------------------------------------------------------------------------- /cdk/kraken/test/integration-tests.test.ts: -------------------------------------------------------------------------------- 1 | import { Workflow } from "cdkactions"; 2 | import { IntegrationTestsJob } from "../src"; 3 | 4 | test("default", () => { 5 | const workflow = new Workflow(undefined as any, "workflow", { 6 | name: "Workflow", 7 | on: "push", 8 | }); 9 | new IntegrationTestsJob(workflow, { 10 | testCommand: "exit 0", 11 | dockerBuildIds: ["id"], 12 | dockerImages: ["image"], 13 | }); 14 | expect(workflow.toGHAction()).toMatchSnapshot(); 15 | }); 16 | 17 | test("no post image publish", () => { 18 | const workflow = new Workflow(undefined as any, "workflow", { 19 | name: "Workflow", 20 | on: "push", 21 | }); 22 | new IntegrationTestsJob(workflow, { 23 | testCommand: "exit 0", 24 | dockerBuildIds: ["id"], 25 | dockerImages: ["image"], 26 | createPostIntegrationPublishJob: false, 27 | }); 28 | expect(workflow.toGHAction()).toMatchSnapshot(); 29 | }); 30 | -------------------------------------------------------------------------------- /cdk/kraken/test/labs-application.test.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | import { LabsApplicationStack } from "../src"; 3 | import { TestingApp } from "./utils"; 4 | 5 | test("default", () => { 6 | const app = TestingApp({ createValidateWorkflow: false }); 7 | new LabsApplicationStack(app, { 8 | djangoProjectName: "example", 9 | dockerImageBaseName: "example", 10 | }); 11 | app.synth(); 12 | expect(fs.readdirSync(app.outdir)).toEqual([ 13 | "cdkactions_build-and-deploy.yaml", 14 | ]); 15 | expect( 16 | fs.readFileSync(`${app.outdir}/cdkactions_build-and-deploy.yaml`, "utf-8") 17 | ).toMatchSnapshot(); 18 | }); 19 | 20 | test("integration tests", () => { 21 | const app = TestingApp({ createValidateWorkflow: false }); 22 | new LabsApplicationStack(app, { 23 | djangoProjectName: "example", 24 | dockerImageBaseName: "example", 25 | integrationTests: true, 26 | }); 27 | app.synth(); 28 | expect(fs.readdirSync(app.outdir)).toEqual([ 29 | "cdkactions_build-and-deploy.yaml", 30 | ]); 31 | expect( 32 | fs.readFileSync(`${app.outdir}/cdkactions_build-and-deploy.yaml`, "utf-8") 33 | ).toMatchSnapshot(); 34 | }); 35 | -------------------------------------------------------------------------------- /cdk/kraken/test/pypi.test.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | import { PyPIPublishStack } from "../src"; 3 | import { TestingApp } from "./utils"; 4 | 5 | test("default", () => { 6 | const app = TestingApp({ createValidateWorkflow: false }); 7 | new PyPIPublishStack(app); 8 | app.synth(); 9 | expect(fs.readdirSync(app.outdir)).toEqual([ 10 | "cdkactions_build-and-publish.yaml", 11 | ]); 12 | expect( 13 | fs.readFileSync(`${app.outdir}/cdkactions_build-and-publish.yaml`, "utf-8") 14 | ).toMatchSnapshot(); 15 | }); 16 | -------------------------------------------------------------------------------- /cdk/kraken/test/react-project.test.ts: -------------------------------------------------------------------------------- 1 | import { Workflow } from "cdkactions"; 2 | import { ReactProject } from "../src"; 3 | 4 | test("default", () => { 5 | const workflow = new Workflow(undefined as any, "workflow", { 6 | name: "Workflow", 7 | on: "push", 8 | }); 9 | new ReactProject(workflow, { 10 | imageName: "example", 11 | }); 12 | expect(workflow.toGHAction()).toMatchSnapshot(); 13 | }); 14 | 15 | test("custom id", () => { 16 | const workflow = new Workflow(undefined as any, "workflow", { 17 | name: "Workflow", 18 | on: "push", 19 | }); 20 | new ReactProject(workflow, { 21 | imageName: "example", 22 | id: "custom", 23 | }); 24 | expect(workflow.toGHAction()).toMatchSnapshot(); 25 | }); 26 | -------------------------------------------------------------------------------- /cdk/kraken/test/react.test.ts: -------------------------------------------------------------------------------- 1 | import { ReactCheckJob } from "../src"; 2 | 3 | test("default", () => { 4 | const dc = new ReactCheckJob(undefined as any, {}); 5 | expect(dc.toGHAction()).toMatchSnapshot(); 6 | }); 7 | 8 | test("different node version", () => { 9 | const dc = new ReactCheckJob(undefined as any, { nodeVersion: "12" }); 10 | expect(dc.toGHAction()).toMatchSnapshot(); 11 | }); 12 | 13 | test("different directory", () => { 14 | const dc = new ReactCheckJob(undefined as any, { path: "frontend" }); 15 | expect(dc.toGHAction()).toMatchSnapshot(); 16 | }); 17 | 18 | test("with overrides", () => { 19 | const dc = new ReactCheckJob(undefined as any, {}, { continueOnError: true }); 20 | expect(dc.toGHAction()).toMatchSnapshot(); 21 | }); 22 | -------------------------------------------------------------------------------- /cdk/kraken/test/utils.test.ts: -------------------------------------------------------------------------------- 1 | import { buildId, buildName } from "../src"; 2 | 3 | test("buildId no suffix", () => { 4 | const id = "id"; 5 | const suffix = ""; 6 | expect(buildId(id, suffix)).toBe(id); 7 | }); 8 | 9 | test("buildId with suffix", () => { 10 | const id = "id"; 11 | const suffix = "abc"; 12 | expect(buildId(id, suffix)).toBe(`${id}-${suffix}`); 13 | }); 14 | 15 | test("buildName no suffix", () => { 16 | const name = "name"; 17 | const id = ""; 18 | expect(buildName(name, id)).toBe(name); 19 | }); 20 | 21 | test("buildName woth suffix", () => { 22 | const name = "name"; 23 | const id = "id"; 24 | expect(buildName(name, id)).toBe(`${name} ${id}`); 25 | }); 26 | -------------------------------------------------------------------------------- /cdk/kraken/test/utils.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | import * as os from "os"; 3 | import * as path from "path"; 4 | import { App, AppProps } from "cdkactions"; 5 | 6 | /** 7 | * A util function returning an instance of App with a outdir set to a temp directory 8 | * @param options AppProps to provide to the new App 9 | */ 10 | export const TestingApp = (options: Partial) => 11 | new App({ 12 | ...options, 13 | outdir: fs.mkdtempSync(path.join(os.tmpdir(), "kraken")), 14 | }); 15 | -------------------------------------------------------------------------------- /cdk/kraken/tsconfig.dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "alwaysStrict": true, 4 | "declaration": true, 5 | "esModuleInterop": true, 6 | "experimentalDecorators": true, 7 | "inlineSourceMap": true, 8 | "inlineSources": true, 9 | "lib": [ 10 | "es2019" 11 | ], 12 | "module": "CommonJS", 13 | "noEmitOnError": false, 14 | "noFallthroughCasesInSwitch": true, 15 | "noImplicitAny": true, 16 | "noImplicitReturns": true, 17 | "noImplicitThis": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "resolveJsonModule": true, 21 | "strict": true, 22 | "strictNullChecks": true, 23 | "strictPropertyInitialization": true, 24 | "stripInternal": true, 25 | "target": "ES2019" 26 | }, 27 | "include": [ 28 | ".projenrc.js", 29 | "src/**/*.ts", 30 | "test/**/*.ts" 31 | ], 32 | "exclude": [ 33 | "node_modules" 34 | ], 35 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"npx projen\"." 36 | } 37 | -------------------------------------------------------------------------------- /cdk/kraken/tsconfig.jest.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "alwaysStrict": true, 4 | "declaration": true, 5 | "experimentalDecorators": true, 6 | "inlineSourceMap": true, 7 | "inlineSources": true, 8 | "lib": [ 9 | "es2018" 10 | ], 11 | "module": "CommonJS", 12 | "noEmitOnError": false, 13 | "noFallthroughCasesInSwitch": true, 14 | "noImplicitAny": true, 15 | "noImplicitReturns": true, 16 | "noImplicitThis": true, 17 | "noUnusedLocals": true, 18 | "noUnusedParameters": true, 19 | "resolveJsonModule": true, 20 | "strict": true, 21 | "strictNullChecks": true, 22 | "strictPropertyInitialization": true, 23 | "stripInternal": true, 24 | "target": "ES2018" 25 | }, 26 | "include": [ 27 | ".projenrc.js", 28 | "src/**/*.ts", 29 | "test/**/*.ts" 30 | ], 31 | "exclude": [ 32 | "node_modules" 33 | ] 34 | } -------------------------------------------------------------------------------- /cdk/kraken/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "rootDir": "src", 4 | "outDir": "lib", 5 | "alwaysStrict": true, 6 | "declaration": true, 7 | "esModuleInterop": true, 8 | "experimentalDecorators": true, 9 | "inlineSourceMap": true, 10 | "inlineSources": true, 11 | "lib": [ 12 | "es2019" 13 | ], 14 | "module": "CommonJS", 15 | "noEmitOnError": false, 16 | "noFallthroughCasesInSwitch": true, 17 | "noImplicitAny": true, 18 | "noImplicitReturns": true, 19 | "noImplicitThis": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "resolveJsonModule": true, 23 | "strict": true, 24 | "strictNullChecks": true, 25 | "strictPropertyInitialization": true, 26 | "stripInternal": true, 27 | "target": "ES2019" 28 | }, 29 | "include": [ 30 | "src/**/*.ts" 31 | ], 32 | "exclude": [], 33 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"npx projen\"." 34 | } 35 | -------------------------------------------------------------------------------- /cdk/kraken/version.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.8.12" 3 | } 4 | -------------------------------------------------------------------------------- /cdk/projen-common.js: -------------------------------------------------------------------------------- 1 | exports.options = { 2 | repository: 'https://github.com/pennlabs/infrastructure.git', 3 | authorName: "Penn Labs", 4 | authorAddress: "contact@pennlabs.org", 5 | authorOrganization: 'Penn Labs', 6 | authorUrl: 'https://pennlabs.org', 7 | buildWorkflow: false, 8 | pullRequestTemplate: false, 9 | releaseWorkflow: false, 10 | stale: false, 11 | githubOptions: { 12 | pullRequestLint: false, 13 | }, 14 | depsUpgrade: false, 15 | dependabot: false, 16 | mergify: false, 17 | compat: false, 18 | dependabot: false, 19 | rebuildBot: false, 20 | clobber: false, 21 | docgen: true, 22 | docsDirectory: 'docs', 23 | license: 'MIT', 24 | licensed: true, 25 | gitignore: ['/docs'], 26 | defaultReleaseBranch: 'master', 27 | tsconfig: { 28 | compilerOptions: { 29 | esModuleInterop: true, 30 | }, 31 | }, 32 | typescriptConfig: { 33 | tsconfigDev: { 34 | compilerOptions: { 35 | esModuleInterop: true, 36 | }, 37 | }, 38 | jestConfig: { 39 | coveragePathIgnorePatterns: ['src/imports'], 40 | }, 41 | }, 42 | prettier: true, 43 | prettierOptions: { 44 | ignoreFile: true, 45 | }, 46 | jestOptions: { 47 | ignorePatterns: ['src/imports'], 48 | }, 49 | eslintOptions: { 50 | ignorePatterns: ['src/imports/*'], 51 | prettier: true, 52 | }, 53 | } 54 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Docker 2 | 3 | This directory contains all the docker images that we maintain for our infrastructure. All of the images are built using [GitHub Actions](https://github.com/features/actions), tagged with their commit sha, and pushed to [Docker Hub](https://hub.docker.com/). 4 | 5 | Note that we're currently in the process of moving our docker images into this repo. So the list below is not exhaustive. 6 | 7 | We maintain the following docker images: 8 | 9 | | Docker Image | Description | 10 | | ------------------------------------------ | --------------------------------------------------------------------------------------- | 11 | | [Django base](django-base) | A base docker image for django | 12 | | [Pg S3 Backup](pg-s3-backup) | A docker image that bundles the aws cli and postgres used to backup our databases daily | 13 | | [Shibboleth SP nginx](shibboleth-sp-nginx) | A docker image that contains a Shibboleth SP running behind nginx | 14 | | [Team sync](team-sync) | A docker image to sync GitHub teams to vault policies | 15 | | [Datadog agent](datadog-agent) | A docker image that bundles extra integrations with the base datadog image | 16 | -------------------------------------------------------------------------------- /docker/django-base/.dockerignore: -------------------------------------------------------------------------------- 1 | # Docker 2 | Dockerfile 3 | .dockerignore 4 | 5 | # git 6 | .github 7 | .git 8 | .gitignore 9 | .gitmodules 10 | **/*.md 11 | LICENSE 12 | -------------------------------------------------------------------------------- /docker/django-base/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION=3.10 2 | FROM python:${PYTHON_VERSION}-slim-buster 3 | 4 | LABEL maintainer="Penn Labs" 5 | 6 | # Install build dependencies 7 | RUN apt-get update && apt-get install --no-install-recommends -y gcc libpq-dev libc-dev \ 8 | && rm -rf /var/lib/apt/lists/* 9 | 10 | # Copy mime definitions 11 | COPY mime.types /etc/mime.types 12 | 13 | # Install uv 14 | COPY --from=ghcr.io/astral-sh/uv@sha256:2381d6aa60c326b71fd40023f921a0a3b8f91b14d5db6b90402e65a635053709 /uv /uvx /bin/ 15 | 16 | # Copy run file 17 | COPY django-run /usr/local/bin/ 18 | 19 | WORKDIR /app/ 20 | 21 | CMD ["/usr/local/bin/django-run"] 22 | -------------------------------------------------------------------------------- /docker/django-base/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Penn Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /docker/django-base/README.md: -------------------------------------------------------------------------------- 1 | # django-base 2 | 3 | [![Docker Pulls](https://img.shields.io/docker/pulls/pennlabs/django-base)](https://hub.docker.com/r/pennlabs/django-base) 4 | 5 | This repository contains a base docker image to use with django. To use it, create a Dockerfile like: 6 | 7 | ```Dockerfile 8 | From pennlabs/django-base 9 | 10 | # Copy project dependencies 11 | COPY Pipfile* /app/ 12 | 13 | # Install project dependencies 14 | RUN pipenv install --system 15 | 16 | # Copy project files 17 | COPY . /app/ 18 | 19 | # Collect static files 20 | RUN python3.7 /app/manage.py collectstatic --noinput 21 | ``` 22 | 23 | ## Tags 24 | Currently, our Penn Labs products use different image tags for django-base. The main reason is that different products are compatible with different versions of python. 25 | 26 | Because of this, each `django-base` is comprised of "Shared" and "Simple" tags. For each new version of `django-base` (associated with a GIT sha), a new "Shared tag" is created. The "Simple tag" is the same as the "Shared tag" but with the python version appended to the end. 27 | 28 | For example, the current "Shared tag" can be `a142aa6975ee293bbc8a09ef0b81998ce7063dd3` and the current "Simple tag" is `a142aa6975ee293bbc8a09ef0b81998ce7063dd3-python3.10`. 29 | 30 | - `[sha]`: Python 3.10.1 (default) 31 | - `[sha]-3.10`: Python 3.10.1 (`penn-courses`) 32 | - `[sha]-3.9`: Python 3.9.14 (`penn-mobile`) 33 | - `[sha]-3.8`: Python 3.8.11 (`penn-clubs`, `office-hours-queue`) 34 | 35 | ## Features 36 | 37 | This docker image contains pipenv and necessary packages to use `mysqlclient` 38 | 39 | ## Build Args 40 | The `django-base` image supports the build-arg `PYTHON_VERSION` which can be used to specify the python version to use. The default is `3.10.1`. -------------------------------------------------------------------------------- /docker/django-base/django-run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Django Migrate 4 | /usr/local/bin/python3 /app/manage.py migrate --noinput 5 | 6 | # Run UWSGI 7 | exec /usr/local/bin/uwsgi --ini /app/setup.cfg --listen 1000 8 | -------------------------------------------------------------------------------- /docker/pg-s3-backup/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:buster-slim as build 2 | 3 | ENV AWS_CLI_VERSION 2.0.27 4 | 5 | # Install aws cli 6 | RUN apt-get update \ 7 | && apt-get install -y curl unzip \ 8 | && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" \ 9 | && unzip awscliv2.zip \ 10 | && ./aws/install --bin-dir /aws-cli-bin/ 11 | 12 | 13 | FROM debian:buster-slim 14 | 15 | LABEL maintainer="Penn Labs" 16 | 17 | # Install postgres tools 18 | RUN apt-get update \ 19 | && apt-get install -y postgresql-client \ 20 | && rm -rf /var/lib/apt/lists/* 21 | 22 | # Copy aws cli binary + config 23 | COPY --from=build /usr/local/aws-cli/ /usr/local/aws-cli/ 24 | COPY --from=build /aws-cli-bin/ /usr/local/bin/ 25 | 26 | # Copy backup script 27 | COPY backup /usr/local/bin/ 28 | 29 | WORKDIR /app/ 30 | 31 | CMD ["/usr/local/bin/backup"] 32 | -------------------------------------------------------------------------------- /docker/pg-s3-backup/README.md: -------------------------------------------------------------------------------- 1 | # Pg S3 Backup 2 | 3 | This docker image backups up all databases in a PostgreSQL cluster to an S3 bucket. 4 | 5 | ## Usage 6 | 7 | The following environment variables need to be defined: 8 | 9 | * AWS_ACCESS_KEY_ID 10 | * AWS_SECRET_ACCESS_KEY 11 | * AWS_DEFAULT_REGION 12 | * S3_BUCKET 13 | * DATABASE_URL 14 | -------------------------------------------------------------------------------- /docker/pg-s3-backup/backup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | # Helper - run a command in the background and check the exit status 6 | run_bg_and_check_exit() { 7 | "$@" & 8 | local pid=$! 9 | wait "$pid" 10 | local exit_status=$? 11 | if [ $exit_status -ne 0 ]; then 12 | echo "Error: command '$@' exited with non-zero status $exit_status" 13 | exit $exit_status 14 | fi 15 | } 16 | 17 | # Get current timestamp 18 | STAMP=`date +"%Y%m%d - %A %d %B %Y @ %H:%M"` 19 | 20 | # List all databases and remove default databases 21 | DATABASES=`psql $DATABASE_URL/postgres -q -t -c 'SELECT datname from pg_database' | grep -v "\(postgres\|rdsadmin\|template0\|template1\)"` 22 | 23 | echo "Starting backup" 24 | 25 | # Dump each database and upload to S3 26 | for DB in $DATABASES 27 | do 28 | (FILENAME="$STAMP - $DB.sql.gz"; 29 | echo "Creating dump of $DB"; 30 | # Use the helper function to run the command in the background and check the exit status 31 | run_bg_and_check_exit pg_dump $DATABASE_URL/$DB | gzip > $DB.dump; 32 | echo "Uploading dump of $DB"; 33 | # Use the helper function to run the command in the background and check the exit status 34 | run_bg_and_check_exit cat $DB.dump | aws s3 cp - "s3://$S3_BUCKET/$STAMP/$FILENAME") & 35 | done 36 | 37 | wait 38 | 39 | echo "Backup finished successfully" 40 | echo "Total data backed up: `du -hc *.dump | grep total | cut -f1`" 41 | -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/.dockerignore: -------------------------------------------------------------------------------- 1 | # Docker 2 | Dockerfile 3 | .dockerignore 4 | 5 | # git 6 | .github 7 | .git 8 | .gitignore 9 | .gitmodules 10 | **/*.md 11 | LICENSE 12 | -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | # Build stage 2 | FROM debian:bookworm-slim AS build 3 | 4 | # Install dependencies 5 | RUN apt-get update \ 6 | && apt-get install --no-install-recommends -y gnupg2 ca-certificates wget git mercurial build-essential lsb-release devscripts fakeroot quilt libssl-dev libpcre2-dev libpcre3-dev zlib1g-dev debhelper libxml2-utils xsltproc libparse-recdescent-perl \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | # Add Nginx repository and install 10 | RUN wget -qO - https://nginx.org/keys/nginx_signing.key | apt-key add - \ 11 | && echo "deb http://nginx.org/packages/debian/ bookworm nginx" > /etc/apt/sources.list.d/nginx.list \ 12 | && apt-get update && apt-get install --no-install-recommends -y nginx \ 13 | && rm -rf /var/lib/apt/lists/* 14 | 15 | # Install pkg-oss 16 | WORKDIR /root/ 17 | RUN mkdir -p /root/nginx-modules/deb/ \ 18 | && wget -qO tip.tar.gz https://hg.nginx.org/pkg-oss/archive/tip.tar.gz \ 19 | && tar -xvf tip.tar.gz && mv pkg-oss-* pkg-oss/ && rm -f tip.tar.gz 20 | 21 | # Build Nginx modules 22 | RUN pkg-oss/build_module.sh --skip-depends -y -o /root/nginx-modules/deb/ -n shibboleth -v `nginx -v 2>&1 | sed -n -e 's|^.*/||p' | tr -d '\n'` https://github.com/nginx-shib/nginx-http-shibboleth.git \ 23 | && pkg-oss/build_module.sh --skip-depends -y -o /root/nginx-modules/deb/ -n headersmore -v `nginx -v 2>&1 | sed -n -e 's|^.*/||p' | tr -d '\n'` https://github.com/openresty/headers-more-nginx-module.git \ 24 | && rm -f /root/nginx-modules/deb/*-dbg_*.deb 25 | 26 | # Production stage 27 | FROM debian:bookworm-slim 28 | 29 | LABEL maintainer="Penn Labs" 30 | 31 | # Install dependencies 32 | RUN apt-get update \ 33 | && apt-get install --no-install-recommends -y gnupg2 wget ca-certificates \ 34 | && rm -rf /var/lib/apt/lists/* 35 | 36 | # Add Nginx repository 37 | RUN wget -qO - https://nginx.org/keys/nginx_signing.key | apt-key add - \ 38 | && echo "deb http://nginx.org/packages/debian/ bookworm nginx" > /etc/apt/sources.list.d/nginx.list 39 | 40 | # Install Shibboleth, Nginx, and Supervisor 41 | RUN apt-get update && apt-get install --no-install-recommends -y libapache2-mod-shib supervisor nginx \ 42 | && apt-get clean \ 43 | && rm -rf /var/lib/apt/lists/* 44 | 45 | # Install Nginx modules 46 | COPY --from=build /root/nginx-modules/deb/*.deb /tmp/ 47 | RUN dpkg -i /tmp/*.deb 48 | 49 | # Copy config files 50 | COPY nginx-default.conf /etc/nginx/conf.d/default.conf 51 | COPY supervisord.conf /etc/supervisor/ 52 | COPY nginx/ /etc/nginx/ 53 | COPY shibd.logger /etc/shibboleth/ 54 | 55 | # Set up Shibboleth directories 56 | RUN mkdir /opt/shibboleth && chown _shibd:_shibd /opt/shibboleth 57 | RUN mkdir /run/shibboleth && chown _shibd:_shibd /run/shibboleth 58 | 59 | # Allow Nginx to access Shibboleth sockets 60 | RUN adduser nginx _shibd 61 | 62 | EXPOSE 80 443 63 | 64 | CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] 65 | -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Penn Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/README.md: -------------------------------------------------------------------------------- 1 | # docker-shibboleth-sp-nginx 2 | 3 | [![Docker Pulls](https://img.shields.io/docker/pulls/pennlabs/shibboleth-sp-nginx)](https://hub.docker.com/r/pennlabs/shibboleth-sp-nginx) 4 | 5 | This docker image contains Nginx with Shibboleth SP 3.0.4 running on Debian Buster (Slim). The image was built following [these instructions](https://github.com/nginx-shib/nginx-http-shibboleth). 6 | 7 | This image is meant to be used as a base image with local changes overriding the default configs. 8 | 9 | Ports 80 and 443 are exposed. 10 | 11 | ## Use as a Base 12 | 13 | To use this image as a base, just add your custom shibboleth and nginx configuration. For example a basic Dockerfile could look like: 14 | 15 | ```Dockerfile 16 | FROM pennlabs/shibboleth-sp-nginx 17 | 18 | COPY shibboleth/ /etc/shibboleth/ 19 | COPY nginx/ /etc/nginx/conf.d/ 20 | ``` 21 | 22 | where `shibboleth` and `nginx` contain your custom configuration files like `shibboleth2.xml` and `default.conf`. 23 | 24 | **WARNING** When using this image as a base, you must generate a new TLS certificate and key for nginx (`cert.pem` and `key.pem`). Copy them to `/etc/nginx/`. 25 | -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/VERSION.txt: -------------------------------------------------------------------------------- 1 | 3.0.4 2 | -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/nginx-default.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80 default_server; 3 | server_name _; 4 | return 301 https://$host$request_uri; 5 | } 6 | 7 | 8 | server { 9 | listen 443 ssl; 10 | server_name _; 11 | ssl_certificate /etc/nginx/cert.pem; 12 | ssl_certificate_key /etc/nginx/key.pem; 13 | 14 | #FastCGI authorizer for Auth Request module 15 | location = /shibauthorizer { 16 | internal; 17 | include fastcgi_params; 18 | fastcgi_pass unix:/opt/shibboleth/shibauthorizer.sock; 19 | } 20 | 21 | #FastCGI responder 22 | location /Shibboleth.sso { 23 | include fastcgi_params; 24 | fastcgi_pass unix:/opt/shibboleth/shibresponder.sock; 25 | } 26 | 27 | #Resources for the Shibboleth error pages. This can be customised. 28 | location /shibboleth-sp { 29 | alias /usr/share/shibboleth/; 30 | } 31 | 32 | #A secured location. Here all incoming requests query the 33 | #FastCGI authorizer. Watch out for performance issues and spoofing. 34 | location /secure { 35 | include shib_clear_headers; 36 | shib_request /shibauthorizer; 37 | shib_request_use_headers on; 38 | proxy_pass https://$host/Shibboleth.sso/Login?target=https%3A%2F%$host%2FShibboleth.sso%2FSession; 39 | } 40 | } -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/nginx/cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFCTCCAvGgAwIBAgIUTE9Cd7XVJauAhwmZV1PSxJGKP/0wDQYJKoZIhvcNAQEL 3 | BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE5MDgwMjA0NDY0MVoXDTIwMDgw 4 | MTA0NDY0MVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF 5 | AAOCAg8AMIICCgKCAgEAuoyG0NYBt9a1/tLaI+SETNbHGE6zwlcB3nBIYf2FgvKp 6 | 8d0NnslMPLxrgXOARB1sfY7pwNUWVkWhunXY2pLQ2yztR9SIXdR72r24MM4m65A5 7 | nTlkU2tsjQ95uicW0DQfmM3Cf4Xf5JTgko+IpyU5MZvRpa2EVJUP0p4Ln97DV2q4 8 | iW/sj8bj05xdexUE7bUgH0EyS9vDUSP4r9YMPL06uP6hzoYFk+GoMmPplqspb6Z3 9 | KtD5FUe7Oe4HCyCuD7aUv/uWlOz6ai8rzyRBh+Ja5GHN1d6BaVs+O2DXsmqhXoA0 10 | Iv2WlHzmP7xYnjdTt6JEwviqu5ncKMLM/paW3W8I/g0sK1nmAe0/SsmwENMrJ64b 11 | A+/x6M721StDZAacn+LA6nv8vAYyajc5t3qG5Ug8qsBh+GIiBvCSGFANGotaZLtH 12 | h6JZLPTzRZ6ZyNrR6i1yNnWTkQLhIIG6BKxM+bOkln9CPnDvb9EURQeO4ntfI2uV 13 | EKk1KjRtatPWusxoqgPZwYYuSpwMr304sZ7sPlqa5oq7neG2+cnVBnAHxPCoDuFU 14 | QTgqS3bavGuU1zBpCnALKQ87vMAJ2+sGU/zbII5cUm5UHVjbynLnLFsgxILZ1wGs 15 | xffw9hgTk0iYVn8ycOUcMW86QvY6hzEvqh4oo03Q+6Y4b526/Q7sO1pdeswXMB8C 16 | AwEAAaNTMFEwHQYDVR0OBBYEFBXEstFoBV+E8E3+TCUPae/SSD86MB8GA1UdIwQY 17 | MBaAFBXEstFoBV+E8E3+TCUPae/SSD86MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI 18 | hvcNAQELBQADggIBAJxbJIkBS0S9/9Z1cxG9BNXCW0MQOW7xsKRlUdxhnlJGm/lu 19 | /Wo8hoRjliYEvy1JxUCeymvDiDz7Xbcy4PvZoB/r7jIxUjAjI3Aw1n0SRPY//w1c 20 | XXfdNFrQ43lR2C6ClC89mD9v1H1Xs7cFpAWzQ+KfdSHSIfUVaq9NJTpeoCz710LX 21 | 1aInRPw/NokpB/A0X23aahBIXlrjqtKveWITmZ3kfxor8q8HQPrPLBevJFxcCFda 22 | hq6S8d3kXDXPZ+NgCG2WbEuTYXHmv1V1Go4aJvje7Pen368bBKfyCLPNUjKi5Tn2 23 | Z98CYP1D3xgPlKOUas8pYEmmgzwA/iBWLtYqCOxGfpBp1y3CwN3U/6nVP6PSLVkS 24 | kdOb+7fClzJFSnyt9mW+Amzt+yv6dhPa4Fq3qLJ618nWepitDbLmNBwn83MHgDUS 25 | Q71k5Eorr6CgACu50MdTN7PuXalZsJ3fBXgDrgako0w/Qn9bNKMEd1bszDk8Kq38 26 | JUBGtgIjEaeM0Ti6KTL9QJImACBTJMXdD2mQxZmBgcaJrwZ4kNF9xWHf452fmnPA 27 | 1R+jxIAMubyk2t9uqySodkjOtz0XM3GKt3LZuzTLWakWNH/XIFRSDC7/rOTBtV4s 28 | zpTxrVieGopsvc0Ic3wFxn6MViDN23X02YG53uhjZfIJ3IL4aELxNGxOKFqM 29 | -----END CERTIFICATE----- 30 | -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | 2 | user nginx; 3 | worker_processes 1; 4 | 5 | error_log /dev/stderr warn; 6 | pid /var/run/nginx.pid; 7 | 8 | load_module "modules/ngx_http_headers_more_filter_module.so"; 9 | load_module "modules/ngx_http_shibboleth_module.so"; 10 | 11 | events { 12 | worker_connections 1024; 13 | } 14 | 15 | http { 16 | include /etc/nginx/mime.types; 17 | default_type application/octet-stream; 18 | 19 | log_format main 'nginx-access $remote_addr - $remote_user [$time_local] "$request" ' 20 | '$status $body_bytes_sent "$http_referer" ' 21 | '"$http_user_agent" "$http_x_forwarded_for"'; 22 | 23 | access_log /dev/stdout main; 24 | 25 | sendfile on; 26 | #tcp_nopush on; 27 | 28 | keepalive_timeout 65; 29 | 30 | #gzip on; 31 | 32 | include /etc/nginx/conf.d/*.conf; 33 | 34 | server_tokens off; 35 | } -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/nginx/shib_clear_headers: -------------------------------------------------------------------------------- 1 | # Ensure that you add directives to clear input headers for *all* attributes 2 | # that your backend application uses. This may also include variations on these 3 | # headers, such as differing capitalisations and replacing hyphens with 4 | # underscores etc -- it all depends on what your application is reading. 5 | # 6 | # Note that Nginx silently drops headers with underscores 7 | # unless the non-default `underscores_in_headers` is enabled. 8 | 9 | # Shib-* doesn't currently work because * isn't (yet) supported 10 | more_clear_input_headers 11 | Auth-Type 12 | Shib-Application-Id 13 | Shib-Authentication-Instant 14 | Shib-Authentication-Method 15 | Shib-Authncontext-Class 16 | Shib-Identity-Provider 17 | Shib-Session-Id 18 | Shib-Session-Index 19 | Remote-User; 20 | 21 | # more_clear_input_headers 22 | # EPPN 23 | # Affiliation 24 | # Unscoped-Affiliation 25 | # Entitlement 26 | # Targeted-Id 27 | # Persistent-Id 28 | # Transient-Name 29 | # Commonname 30 | # DisplayName 31 | # Email 32 | # OrganizationName; 33 | -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/shibd.logger: -------------------------------------------------------------------------------- 1 | # set overall behavior 2 | log4j.rootCategory=INFO, shibd_log 3 | 4 | # fairly verbose for DEBUG, so generally leave at INFO 5 | log4j.category.XMLTooling.XMLObject=INFO 6 | log4j.category.XMLTooling.KeyInfoResolver=INFO 7 | log4j.category.Shibboleth.IPRange=INFO 8 | log4j.category.Shibboleth.PropertySet=INFO 9 | 10 | # raise for low-level tracing of SOAP client HTTP/SSL behavior 11 | log4j.category.XMLTooling.libcurl=INFO 12 | 13 | # useful categories to tune independently: 14 | # 15 | # tracing of SAML messages and security policies 16 | #log4j.category.OpenSAML.MessageDecoder=DEBUG 17 | #log4j.category.OpenSAML.MessageEncoder=DEBUG 18 | #log4j.category.OpenSAML.SecurityPolicyRule=DEBUG 19 | #log4j.category.XMLTooling.SOAPClient=DEBUG 20 | # interprocess message remoting 21 | #log4j.category.Shibboleth.Listener=DEBUG 22 | # mapping of requests to applicationId 23 | #log4j.category.Shibboleth.RequestMapper=DEBUG 24 | # high level session cache operations 25 | #log4j.category.Shibboleth.SessionCache=DEBUG 26 | # persistent storage and caching 27 | #log4j.category.XMLTooling.StorageService=DEBUG 28 | 29 | # logs XML being signed or verified if set to DEBUG 30 | log4j.category.XMLTooling.Signature.Debugger=INFO, sig_log 31 | log4j.additivity.XMLTooling.Signature.Debugger=false 32 | log4j.ownAppenders.XMLTooling.Signature.Debugger=true 33 | 34 | # the tran log blocks the "default" appender(s) at runtime 35 | # Level should be left at INFO for this category 36 | log4j.category.Shibboleth-TRANSACTION=INFO, tran_log 37 | log4j.additivity.Shibboleth-TRANSACTION=false 38 | log4j.ownAppenders.Shibboleth-TRANSACTION=true 39 | 40 | # uncomment to suppress particular event types 41 | #log4j.category.Shibboleth-TRANSACTION.AuthnRequest=WARN 42 | #log4j.category.Shibboleth-TRANSACTION.Login=WARN 43 | #log4j.category.Shibboleth-TRANSACTION.Logout=WARN 44 | 45 | # define the appenders 46 | 47 | log4j.appender.shibd_log=org.apache.log4j.RollingFileAppender 48 | log4j.appender.shibd_log.fileName=/dev/stdout 49 | log4j.appender.shibd_log.maxFileSize=0 50 | log4j.appender.shibd_log.maxBackupIndex=0 51 | log4j.appender.shibd_log.layout=org.apache.log4j.PatternLayout 52 | log4j.appender.shibd_log.layout.ConversionPattern=shib-shibd %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n 53 | 54 | log4j.appender.tran_log=org.apache.log4j.RollingFileAppender 55 | log4j.appender.tran_log.fileName=/dev/stdout 56 | log4j.appender.tran_log.maxFileSize=0 57 | log4j.appender.tran_log.maxBackupIndex=0 58 | log4j.appender.tran_log.layout=org.apache.log4j.PatternLayout 59 | log4j.appender.tran_log.layout.ConversionPattern=shib-transaction %d{%Y-%m-%d %H:%M:%S}|%c|%m%n 60 | 61 | log4j.appender.sig_log=org.apache.log4j.FileAppender 62 | log4j.appender.sig_log.fileName=/dev/stdout 63 | log4j.appender.sig_log.maxFileSize=0 64 | log4j.appender.sig_log.maxBackupIndex=0 65 | log4j.appender.sig_log.layout=org.apache.log4j.PatternLayout 66 | log4j.appender.sig_log.layout.ConversionPattern=shib-signature %m 67 | -------------------------------------------------------------------------------- /docker/shibboleth-sp-nginx/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | logfile=/var/log/supervisor/supervisord.log 4 | pidfile=/var/run/supervisord.pid 5 | 6 | [fcgi-program:shibauthorizer] 7 | command=/usr/lib/x86_64-linux-gnu/shibboleth/shibauthorizer 8 | socket=unix:///opt/shibboleth/shibauthorizer.sock 9 | socket_owner=_shibd:_shibd 10 | socket_mode=0660 11 | user=_shibd 12 | stdout_logfile=/dev/stdout 13 | redirect_stderr=true 14 | stdout_logfile_maxbytes=0 15 | autorestart = true 16 | 17 | [fcgi-program:shibresponder] 18 | command=/usr/lib/x86_64-linux-gnu/shibboleth/shibresponder 19 | socket=unix:///opt/shibboleth/shibresponder.sock 20 | socket_owner=_shibd:_shibd 21 | socket_mode=0660 22 | user=_shibd 23 | stdout_logfile=/dev/stdout 24 | redirect_stderr=true 25 | stdout_logfile_maxbytes=0 26 | autorestart = true 27 | 28 | [program:shibd] 29 | command=/usr/sbin/shibd -f -F 30 | stdout_logfile=/dev/stdout 31 | redirect_stderr=true 32 | stdout_logfile_maxbytes=0 33 | autorestart=true 34 | 35 | [program:nginx] 36 | command=/usr/sbin/nginx -g "daemon off;" 37 | stdout_logfile=/dev/stdout 38 | redirect_stderr=true 39 | stdout_logfile_maxbytes=0 40 | autorestart=true 41 | -------------------------------------------------------------------------------- /docker/team-sync/.dockerignore: -------------------------------------------------------------------------------- 1 | # Docker 2 | Dockerfile 3 | .dockerignore 4 | 5 | # git 6 | .github 7 | .git 8 | .gitignore 9 | .gitmodules 10 | **/*.md 11 | LICENSE 12 | -------------------------------------------------------------------------------- /docker/team-sync/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /docker/team-sync/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-slim 2 | 3 | LABEL maintainer="Penn Labs" 4 | 5 | RUN apt-get update && apt-get install --no-install-recommends -y gcc wget unzip ca-certificates \ 6 | && rm -rf /var/lib/apt/lists/* 7 | 8 | RUN pip install pipenv 9 | 10 | RUN wget -qO bw.zip "https://github.com/bitwarden/cli/releases/download/v1.15.1/bw-linux-1.15.1.zip" \ 11 | && unzip bw.zip && rm -f bw.zip && chmod +x bw && mv bw /usr/local/bin 12 | WORKDIR /app/ 13 | 14 | COPY Pipfile* /app/ 15 | 16 | RUN pipenv install --system 17 | 18 | COPY . /app/ 19 | 20 | CMD ["/usr/local/bin/python3", "sync/sync.py"] 21 | -------------------------------------------------------------------------------- /docker/team-sync/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Penn Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docker/team-sync/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | black = "==24.2.0" 8 | flake8 = "*" 9 | flake8-absolute-import = "*" 10 | flake8-isort = "*" 11 | flake8-quotes = "*" 12 | 13 | [packages] 14 | hvac = "*" 15 | jinja2 = "*" 16 | pygithub = "*" 17 | boto3 = "*" 18 | airtable-python-wrapper = "*" 19 | pyotp = "*" 20 | 21 | [requires] 22 | python_version = "3" 23 | -------------------------------------------------------------------------------- /docker/team-sync/README.md: -------------------------------------------------------------------------------- 1 | # Docker Team Sync 2 | 3 | [![Docker Pulls](https://img.shields.io/docker/pulls/pennlabs/team-sync)](https://hub.docker.com/r/pennlabs/team-sync) 4 | 5 | This repository contains a docker image that performs various syncing updates related to Penn Labs' Github teams. The image dynamically loads python files in `sync/modules` and executes their `sync` method. 6 | -------------------------------------------------------------------------------- /docker/team-sync/setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 100 3 | exclude = .venv 4 | inline-quotes = double 5 | 6 | [isort] 7 | known_first_party = sync 8 | line_length = 100 9 | lines_after_imports = 2 10 | multi_line_output = 3 11 | include_trailing_comma = True 12 | use_parentheses = True 13 | 14 | [coverage:run] 15 | omit = */tests/*, */.venv/* 16 | source = . 17 | -------------------------------------------------------------------------------- /docker/team-sync/sync/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pennlabs/infrastructure/26283081d4cffb644593c7f7fa7da28f8c62ab57/docker/team-sync/sync/__init__.py -------------------------------------------------------------------------------- /docker/team-sync/sync/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pennlabs/infrastructure/26283081d4cffb644593c7f7fa7da28f8c62ab57/docker/team-sync/sync/modules/__init__.py -------------------------------------------------------------------------------- /docker/team-sync/sync/modules/platform.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import requests 4 | 5 | 6 | PLATFORM_API_KEY = os.environ.get("PLATFORM_API_KEY", "") 7 | 8 | 9 | def sync(teams, users): 10 | """ 11 | Sync team leads from GitHub to admin permissions on platform 12 | """ 13 | # Dictionary of pennkey to list of admin permissions 14 | content = {} 15 | # Hub@Penn needs to be pre-populated 16 | team_slugs = ["hub_admin"] 17 | 18 | # Team Leads are admins of their products 19 | for team in teams["leads"]: 20 | team_slug = team.slug[:-6] # Trim -leads 21 | slug = f"{team_slug.replace('-', '_')}_admin" 22 | team_slugs.append(slug) 23 | for member in team.get_members(): 24 | gh_username = member.login.lower() 25 | pennkey = users.get(gh_username, {}).get("pennkey", None) 26 | if pennkey: 27 | content.setdefault(pennkey, []).append(slug) 28 | # If user is a lead of clubs, also make h@p admin 29 | if slug == "penn_clubs_admin": 30 | content[pennkey].append("hub_admin") 31 | 32 | # Directors are admins on all products 33 | for team in teams["directors"]: 34 | for member in team.get_members(): 35 | gh_username = member.login.lower() 36 | pennkey = users.get(gh_username, {}).get("pennkey", None) 37 | if pennkey: 38 | content[pennkey] = team_slugs 39 | 40 | headers = {"Authorization": f"Api-Key {PLATFORM_API_KEY}"} 41 | requests.post( 42 | "https://platform.pennlabs.org/accounts/productadmin/", json=content, headers=headers 43 | ) 44 | return 45 | -------------------------------------------------------------------------------- /docker/team-sync/sync/modules/user-policy.hcl.j2: -------------------------------------------------------------------------------- 1 | # List auth methods 2 | path "sys/auth" 3 | { 4 | capabilities = ["read"] 5 | } 6 | 7 | path "secrets/" 8 | { 9 | capabilities = ["list"] 10 | } 11 | 12 | path "secrets/*" 13 | { 14 | capabilities = ["list"] 15 | } 16 | 17 | path "sys/mounts/*" 18 | { 19 | capabilities = ["list", "read"] 20 | } 21 | 22 | # need to use `data/` because we're on secrets engine v2 23 | # `data/` will not appear in the actual usage of vault since most clients 24 | # intelligently add it in. it only appears in the policy 25 | # production/default and production/staging map to the default and staging 26 | # namespaces of our production cluster 27 | {% for repo in repos %} 28 | path "secrets/data/production/default/{{ repo }}-*" 29 | { 30 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 31 | } 32 | 33 | path "secrets/data/production/default/{{ repo }}" 34 | { 35 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 36 | } 37 | 38 | path "secrets/data/production/staging/{{ repo }}-*" 39 | { 40 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 41 | } 42 | 43 | path "secrets/data/production/staging/{{ repo }}" 44 | { 45 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 46 | } 47 | {% endfor %} 48 | -------------------------------------------------------------------------------- /docker/team-sync/sync/modules/vault.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | import hvac 5 | from jinja2 import Template 6 | 7 | 8 | def sync(teams, users): 9 | session = boto3.Session() 10 | credentials = session.get_credentials() 11 | client = hvac.Client(url=os.getenv("VAULT_ADDR")) 12 | client.auth.aws.iam_login(credentials.access_key, credentials.secret_key, credentials.token) 13 | if not client.sys.is_sealed(): 14 | # Apply normal team policies 15 | with open("sync/modules/user-policy.hcl.j2") as f: 16 | t = Template(f.read()) 17 | for team in teams["leads"]: 18 | base_team_slug = team.slug.replace("-leads", "") 19 | repos = [x.name for x in team.get_repos()] 20 | pol = t.render(team_name=base_team_slug, repos=repos) 21 | client.sys.create_or_update_policy(name=base_team_slug, policy=pol) 22 | client.auth.github.map_team(team_name=team.slug, policies=[base_team_slug]) 23 | else: 24 | print("vault: Vault sealed. Stopping.") 25 | exit(1) 26 | -------------------------------------------------------------------------------- /docker/team-sync/sync/sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import pkgutil 4 | 5 | from airtable import Airtable 6 | from github import Github 7 | 8 | 9 | AIRTABLE_BASE_KEY = os.environ.get("AIRTABLE_BASE_KEY") 10 | AIRTABLE_API_KEY = os.environ.get("AIRTABLE_API_KEY") 11 | airtable = Airtable(AIRTABLE_BASE_KEY, "Roster", AIRTABLE_API_KEY) 12 | 13 | 14 | def get_user_info(): 15 | """ 16 | Create a dictionary of GitHub usernames to PennKeys and emails. 17 | """ 18 | 19 | roster = airtable.get_all(view="Platform Sync", fields=["Github", "PennKey", "Email"]) 20 | data = {} 21 | for record in roster: 22 | fields = record["fields"] 23 | if "Github" in fields and "PennKey" in fields: 24 | github_url = fields["Github"].lower() 25 | pennkey = fields["PennKey"].lower() 26 | email = fields["Email"].lower() 27 | if github_url.startswith("https://github.com/"): 28 | github_id = github_url.split("/")[3] 29 | data[github_id] = {"pennkey": pennkey, "email": email} 30 | return data 31 | 32 | 33 | def run(): 34 | g = Github(os.getenv("GITHUB_TOKEN")) 35 | 36 | # Generate dictionary of teams 37 | teams = {} 38 | for team in g.get_organization("pennlabs").get_teams(): 39 | if team.name == "Alumni": # Found an Alumni Team 40 | teams.setdefault("alumni", []).append(team) 41 | elif team.name == "Directors": # Found a Director Team 42 | teams.setdefault("directors", []).append(team) 43 | elif team.slug.endswith("-leads"): # Found a Lead Team 44 | teams.setdefault("leads", []).append(team) 45 | else: # Found a Regular Team 46 | teams.setdefault("members", []).append(team) 47 | 48 | # Generate dictionary of GitHub usernames to PennKey 49 | users = get_user_info() 50 | 51 | # Dynamically find each module and run its sync method 52 | location = os.path.join(os.path.dirname(os.path.abspath(__file__)), "modules") 53 | for finder, name, _ in pkgutil.walk_packages([location]): 54 | try: 55 | module = finder.find_module(name).load_module(name) 56 | module.sync(teams, users) 57 | except (AttributeError, TypeError) as e: 58 | print(f"Could not execute module '{name}'. The following exception occurred: {e}") 59 | 60 | 61 | if __name__ == "__main__": 62 | run() 63 | -------------------------------------------------------------------------------- /docker/waypoint/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | LABEL maintainer="Penn Labs" 4 | 5 | 6 | # --- systems setup 7 | RUN apt-get update && apt-get install --no-install-recommends -y gcc libpq-dev libc-dev 8 | 9 | RUN apt-get update && apt-get install --no-install-recommends -y curl git \ 10 | sudo python3-dev python3-pip python3-venv libpq-dev postgresql-client nano \ 11 | htop vim net-tools procps sysstat strace tcpdump lsof \ 12 | && rm -rf /var/lib/apt/lists/* 13 | 14 | # Install locales and configure the correct locale 15 | RUN apt-get update && apt-get install -y locales 16 | RUN locale-gen en_US.UTF-8 17 | RUN update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 18 | 19 | RUN pip3 install uv pipenv 20 | 21 | # --- labs setup 22 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y tzdata \ 23 | && ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime \ 24 | && dpkg-reconfigure -f noninteractive tzdata 25 | 26 | RUN apt-get update && apt-get install -y postgresql postgresql-contrib redis-server systemd 27 | 28 | # Install nvm, node, and npm, yarn 29 | ENV NVM_DIR=/usr/local/nvm 30 | ENV NODE_VERSION=20 31 | 32 | RUN mkdir -p $NVM_DIR && \ 33 | curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash && \ 34 | . $NVM_DIR/nvm.sh && \ 35 | nvm install $NODE_VERSION && \ 36 | nvm alias default $NODE_VERSION && \ 37 | nvm use default 38 | 39 | RUN . $NVM_DIR/nvm.sh && npm install -g yarn 40 | 41 | RUN mkdir /opt/waypoint/ 42 | RUN mkdir /labs/ 43 | 44 | # For each product, setup the python environment 45 | COPY waypoint-init /opt/waypoint/waypoint-init 46 | RUN chmod +x /opt/waypoint/waypoint-init 47 | RUN /opt/waypoint/waypoint-init 48 | 49 | # start redis and postgres service 50 | COPY database-init /opt/waypoint/database-init 51 | RUN chmod +x /opt/waypoint/database-init 52 | 53 | RUN mkdir -p /opt/waypoint/cli 54 | COPY . /opt/waypoint/cli 55 | WORKDIR /opt/waypoint/cli 56 | RUN pip3 install . 57 | 58 | WORKDIR / 59 | 60 | # Copy .bashrc 61 | COPY waypoint-bashrc /root/.bashrc 62 | 63 | # expose ports 64 | EXPOSE 5432 65 | EXPOSE 3000 66 | EXPOSE 8000 67 | -------------------------------------------------------------------------------- /docker/waypoint/database-init: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # Start PostgreSQL & Redis 5 | sudo service postgresql start 6 | redis-server --daemonize yes 7 | 8 | # Create DBs (unless made) 9 | PG_USER="postgres" 10 | PG_PASSWORD="postgres" 11 | PG_DATABASE="postgres" 12 | sudo -u postgres psql -v ON_ERROR_STOP=1 --username "$PG_USER" --dbname "$PG_DATABASE" <<-EOSQL 13 | ALTER USER $PG_USER WITH PASSWORD '$PG_PASSWORD'; 14 | EOSQL 15 | 16 | # Product specific 17 | # This is needed here rather than in init because of the way penn-courses interacts with both of these DBs 18 | sudo -u postgres psql -v ON_ERROR_STOP=1 --username "$PG_USER" --dbname "$PG_DATABASE" <<-EOSQL 19 | DO \$\$ 20 | BEGIN 21 | IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'penn-courses') THEN 22 | CREATE USER "penn-courses" WITH PASSWORD '$PG_PASSWORD'; 23 | ALTER USER "penn-courses" WITH SUPERUSER; 24 | END IF; 25 | END 26 | \$\$; 27 | EOSQL 28 | 29 | echo "PostgreSQL user password updated successfully." 30 | -------------------------------------------------------------------------------- /docker/waypoint/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | INSTALL_DIR="/usr/local/bin" 4 | WAYPOINT_VERSION="v1.0.3" 5 | GITHUB_ORG="pennlabs" 6 | REPO_NAME="infrastructure" 7 | 8 | OS=$(uname -s | tr '[:upper:]' '[:lower:]') 9 | ARCH=$(uname -m) 10 | BINARY_SUFFIX="" 11 | CWD=$(pwd) 12 | 13 | case "$OS" in 14 | linux*) 15 | case "$ARCH" in 16 | x86_64|amd64) 17 | BINARY_SUFFIX="-linux-x86_64" 18 | ;; 19 | *) 20 | echo "Unsupported Linux architecture: $ARCH" 21 | exit 1 22 | ;; 23 | esac 24 | ;; 25 | darwin*) 26 | case "$ARCH" in 27 | arm64|aarch64) 28 | BINARY_SUFFIX="-macos-arm64" 29 | ;; 30 | x86_64|amd64) 31 | BINARY_SUFFIX="-macos-x86_64" 32 | ;; 33 | *) 34 | echo "Unsupported macOS architecture: $ARCH" 35 | exit 1 36 | ;; 37 | esac 38 | ;; 39 | *) 40 | echo "Unsupported operating system: $OS" 41 | exit 1 42 | ;; 43 | esac 44 | 45 | TMP_DIR=$(mktemp -d) 46 | cd $TMP_DIR 47 | 48 | echo "Downloading Waypoint for $OS $ARCH..." 49 | curl -L "https://github.com/${GITHUB_ORG}/${REPO_NAME}/releases/download/${WAYPOINT_VERSION}/waypoint-client${BINARY_SUFFIX}" -o waypoint-client 50 | 51 | chmod +x waypoint-client 52 | sudo mv waypoint-client $INSTALL_DIR/ 53 | 54 | echo "Waypoint client installed successfully!" 55 | echo "Run 'waypoint-client configure' to get started." 56 | echo "Run 'waypoint-client --help' to see available commands." 57 | 58 | rm -rf $TMP_DIR 59 | 60 | cd $CWD 61 | -------------------------------------------------------------------------------- /docker/waypoint/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup( 5 | name="waypoint", 6 | version="1.0.3", 7 | package_dir={"": "src"}, 8 | py_modules=["main", "waypoint_client"], 9 | install_requires=[], 10 | entry_points={ 11 | "console_scripts": [ 12 | "waypoint=main:main", 13 | "waypoint-client=waypoint_client:main", 14 | ], 15 | }, 16 | author="Penn Labs", 17 | author_email="admin@pennlabs.org", 18 | description="Waypoint development environment manager", 19 | python_requires=">=3.6", 20 | ) 21 | -------------------------------------------------------------------------------- /docker/waypoint/src/__init__.py: -------------------------------------------------------------------------------- 1 | """Waypoint package""" 2 | -------------------------------------------------------------------------------- /docker/waypoint/waypoint-bashrc: -------------------------------------------------------------------------------- 1 | waypoint-switch-shell-context() { 2 | if [[ -L /opt/waypoint/current ]]; then 3 | symlink_path=$(readlink /opt/waypoint/current) 4 | PRODUCT_NAME=$(basename "$symlink_path") 5 | 6 | cd "/labs/$PRODUCT_NAME" 7 | source "/opt/waypoint/$PRODUCT_NAME/venv/bin/activate" 8 | else 9 | echo 'Run "waypoint switch " to jump into a product environment.' 10 | fi 11 | } 12 | 13 | waypoint() { 14 | command waypoint $@ 15 | 16 | # For waypoint switch, cd and source the venv in the user's shell 17 | if [[ "$1" == "switch" || "$1" == "start" ]]; then 18 | waypoint-switch-shell-context 19 | fi 20 | } 21 | alias wp=waypoint 22 | 23 | source $NVM_DIR/nvm.sh 24 | echo 'Welcome to Waypoint!' 25 | echo 'Run "waypoint services" to start services.' 26 | waypoint-switch-shell-context -------------------------------------------------------------------------------- /docker/waypoint/waypoint-init: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | declare -A PRODUCT_TO_SHA 4 | declare -A PRODUCT_TO_NODE_VERSION 5 | 6 | PRODUCT_TO_SHA=( 7 | ["office-hours-queue"]="524282d029a330b59158e80299e3be23988f1765" 8 | ["penn-clubs"]="1bcd3ce4f3040395e081989df81fd3105047b24b" 9 | ["penn-mobile"]="f0a65d07764f4e3bce56b3f5349c0b556d708780" 10 | ["penn-courses"]="01e3dac5f5465ad39f810b2c81969f0165c70dcf" 11 | ["platform"]="42c56ff509f60de5389262a9de5e38faf1d9aac2" 12 | ) 13 | 14 | PRODUCT_TO_NODE_VERSION=( 15 | ["office-hours-queue"]="22" 16 | ["penn-clubs"]="20" 17 | ["penn-mobile"]="22" 18 | ["penn-courses"]="22" 19 | ["platform"]="22" 20 | ) 21 | 22 | 23 | for product in "${!PRODUCT_TO_SHA[@]}"; do 24 | echo "Installing dependencies for $product" 25 | sha="${PRODUCT_TO_SHA[$product]}" 26 | PRODUCT_PATH="/opt/waypoint/$product" 27 | mkdir -p "$PRODUCT_PATH" 28 | 29 | curl -o $PRODUCT_PATH/Pipfile "https://raw.githubusercontent.com/pennlabs/$product/$sha/backend/Pipfile" 30 | curl -o $PRODUCT_PATH/Pipfile.lock "https://raw.githubusercontent.com/pennlabs/$product/$sha/backend/Pipfile.lock" 31 | 32 | uv venv "$PRODUCT_PATH/venv" --python 3.11 --prompt "$product" 33 | cd $PRODUCT_PATH 34 | source "$PRODUCT_PATH/venv/bin/activate" && pipenv requirements --dev > requirements.txt && uv pip install -r $PRODUCT_PATH/requirements.txt 35 | done 36 | 37 | echo "Setup complete!" 38 | -------------------------------------------------------------------------------- /docs/secrets/Github.md: -------------------------------------------------------------------------------- 1 | # Github 2 | Most of our secrets are stored as Github Organization Secrets. 3 | 4 | ## How to add Github secrets 5 | Most Github org secrets are added manually, although some are specified via terraform resources in [github.tf](https://github.com/pennlabs/infrastructure/blob/master/terraform/github.tf) file. 6 | 7 | Terraform knows what the value of these secrets are by referencing data/resources that do have access to knowing these values. For example, finding the AWS access key using `data.aws_caller_identity.current.account_id`. 8 | 9 | > For our private repositories (e.g. ocwp), it is not possible to access Github Org Secrets with the free tier. Thus, these repos typically have some org secrets copied over to the repository level ([relevant PR](https://github.com/pennlabs/infrastructure/pull/154)). 10 | 11 | By doing `terraform apply`, the secrets are created on Github. 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /docs/secrets/Vault.md: -------------------------------------------------------------------------------- 1 | # Vault 2 | [Vault](https://www.vaultproject.io/) is a secret manager provided by HashiCorp. We use the open-source version of Vault at Labs. Typically, there are two ways to configure Vault. You can either use a *library* to retrieve the secret data, or create a *side car* to extract your secrets and feed them into the application. We uses the latter method. 3 | 4 | ## Where is Vault hosted? 5 | We host Vault at [vault.pennlabs.org](https://vault.pennlabs.org). This is done by terraform, with Vault's configuration specified [here](https://github.com/pennlabs/infrastructure/blob/master/terraform/vault.tf). 6 | 7 | ## What is Vault hosting? 8 | ### Values in Vault 9 | Secrets in Vault are scoped by their paths. 10 | 11 | - [*Cubbyhole*](https://www.vaultproject.io/docs/secrets/cubbyhole 12 | ): Currently, our cubbyhole has nothing inside it, but it's enabled by default. 13 | - [*KV*](https://www.vaultproject.io/docs/secrets/kv/kv-v2): Key Value storage under path `secrets/`. 14 | 15 | ## How to update Vault secrets? 16 | Secrets in Vault can be updated by: 17 | 1. Updating the terraform files: `vault.tf` (database secrets) 18 | 2. Editing Vault directly at [vault.pennlabs.org](https://vault.pennlabs.org) 19 | 20 | Also, Vault allow for *versioned secrets*, which is a pretty neato way to say that you can view & revert them back to previous versions via the Vault UI. 21 | 22 | ## Terraform Modules 23 | ### [Vault module](../../terraform/modules/vault) 24 | Configures Vault itself. 25 | 26 | An example configuration includes updating [Vault policies](../../terraform/modules/vault/policies). Adding a new policy requires: 27 | 1. Add a new policy file to the [Vault policies](../../terraform/modules/vault/policies) folder. See [`admin.hcl`](../../terraform/modules/vault/policies/admin.hcl) for exmaple. 28 | 2. Modify vault terraform files to include the new policy by declaring a new [`resource "vault_policy"`](../../terraform/modules/vault/main.tf#L21-L27) block. 29 | 3. Apply your changes. 30 | 31 | ### [Vault Flush module](../../terraform/modules/vault_flush) 32 | This module allows you to create/update vault secrets by specifying a path to the secret and the new/updated value. -------------------------------------------------------------------------------- /docs/secrets/img/secrets-flow-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pennlabs/infrastructure/26283081d4cffb644593c7f7fa7da28f8c62ab57/docs/secrets/img/secrets-flow-dark.png -------------------------------------------------------------------------------- /docs/secrets/img/secrets-flow-light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pennlabs/infrastructure/26283081d4cffb644593c7f7fa7da28f8c62ab57/docs/secrets/img/secrets-flow-light.png -------------------------------------------------------------------------------- /grafana-dashboards/README.md: -------------------------------------------------------------------------------- 1 | # Grafana Dashboards 2 | 3 | This directory contains various Grafana Dashboard that we created. 4 | 5 | Internally, we have Grafana set up to directly download the dashboard from Github. 6 | 7 | ## Dashboards 8 | 9 | * **Traefik 1.7 Dashboard** - a dashboard to monitor the various traefik 1.7 instances we have. Based on [this dashboard](https://grafana.com/grafana/dashboards/4475) 10 | * **Pod Dashboard** - a dashboard to monitor the status of all the pods within our clusters. 11 | * **Pod Alerting Dashboard** - a dashboard to alert us when our pods exceed normal conditions. This dashboard is required because Grafana currently doesn't allow for variable datasources within an alert. 12 | * **Cert Manager Dashboard** - a dashboard to see the status of our TLS certificates. Based on [this dashboard](https://grafana.com/grafana/dashboards/11001) 13 | -------------------------------------------------------------------------------- /renovate/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "description": "Default preset for Penn Labs repositories", 4 | "extends": [ 5 | "github>whitesource/merge-confidence:beta", 6 | ":separateMajorReleases", 7 | ":combinePatchMinorReleases", 8 | ":ignoreUnstable", 9 | ":prImmediately", 10 | ":updateNotScheduled", 11 | ":automergeDisabled", 12 | ":ignoreModulesAndTests", 13 | ":prHourlyLimit2", 14 | "group:monorepos", 15 | "group:recommended", 16 | "group:allNonMajor", 17 | "helpers:disableTypesNodeMajor", 18 | "workarounds:all", 19 | "schedule:weekly" 20 | ], 21 | "rangeStrategy": "bump", 22 | "prConcurrentLimit": 5, 23 | "rebaseWhen": "behind-base-branch", 24 | "labels": [ 25 | "dependencies" 26 | ], 27 | "packageRules": [ 28 | { 29 | "matchFiles": [ 30 | ".github/cdk/package.json" 31 | ], 32 | "postUpgradeTasks": { 33 | "commands": [ 34 | "cd .github/cdk && yarn build" 35 | ], 36 | "fileFilters": [ 37 | ".github/workflows/cdkactions_*.yaml" 38 | ] 39 | } 40 | }, 41 | { 42 | "matchUpdateTypes": [ 43 | "major" 44 | ], 45 | "addLabels": [ 46 | "major" 47 | ], 48 | "schedule": "at any time" 49 | }, 50 | { 51 | "matchCurrentVersion": "< 1.0.0", 52 | "separateMinorPatch": true, 53 | "addLabels": [ 54 | "pre-1-0-0" 55 | ], 56 | "minor": { 57 | "groupName": null, 58 | "groupSlug": null, 59 | "addLabels": [ 60 | "major" 61 | ], 62 | "schedule": "at any time" 63 | }, 64 | "patch": { 65 | "groupName": "all non-major dependencies", 66 | "groupSlug": "all-minor-patch" 67 | } 68 | } 69 | ], 70 | "github-actions": { 71 | "enabled": false 72 | }, 73 | "terraform": { 74 | "enabled": false 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /terraform/bastion.tf: -------------------------------------------------------------------------------- 1 | // IAM 2 | resource "aws_iam_instance_profile" "bastion" { 3 | name = "bastion" 4 | role = aws_iam_role.kubectl.name 5 | } 6 | 7 | // EC2 Instance 8 | resource "aws_instance" "bastion" { 9 | // Ubuntu 20.04 Server 10 | ami = "ami-042e8287309f5df03" 11 | instance_type = "t3.nano" 12 | subnet_id = module.vpc.public_subnets[0] 13 | vpc_security_group_ids = [aws_security_group.bastion.id] 14 | iam_instance_profile = aws_iam_instance_profile.bastion.name 15 | key_name = aws_key_pair.admin.key_name 16 | user_data = templatefile("files/bastion/user_data.sh", { 17 | CONTAIN_EXEC_ENTRY = file("files/bastion/container_exec_entry.sh") 18 | CONTAIN_EXEC = file("files/bastion/container_exec.sh") 19 | SSH_AUTHORIZED_KEYS = file("files/bastion/ssh_authorized_keys") 20 | }) 21 | tags = { 22 | Name = "Bastion" 23 | created-by = "terraform" 24 | } 25 | } 26 | 27 | resource "aws_security_group" "bastion" { 28 | name = "bastion" 29 | description = "Allow TLS inbound traffic" 30 | vpc_id = module.vpc.vpc_id 31 | 32 | // SSH 33 | ingress { 34 | description = "SSH" 35 | from_port = 22 36 | to_port = 22 37 | protocol = "tcp" 38 | cidr_blocks = ["0.0.0.0/0"] 39 | } 40 | 41 | // Access to internet (can't restrict to just the cluster 42 | // because we need to download tools on first startup) 43 | egress { 44 | from_port = 0 45 | to_port = 0 46 | protocol = "-1" 47 | cidr_blocks = ["0.0.0.0/0"] 48 | } 49 | 50 | tags = { 51 | Name = "Bastion" 52 | created-by = "terraform" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /terraform/db-backup.tf: -------------------------------------------------------------------------------- 1 | data "aws_iam_policy_document" "db-backup" { 2 | statement { 3 | actions = ["s3:PutObject", "s3:Get*"] 4 | resources = [ 5 | "arn:aws:s3:::sql.pennlabs.org/*", 6 | "arn:aws:s3:::sql.pennlabs.org" 7 | ] 8 | } 9 | } 10 | 11 | resource "aws_iam_role_policy" "db-backup" { 12 | name = "db-backup" 13 | role = module.iam-products["db-backup"].role-id 14 | 15 | policy = data.aws_iam_policy_document.db-backup.json 16 | } 17 | -------------------------------------------------------------------------------- /terraform/files/bastion/container_exec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Need to escape all $ because of TF formatting 4 | # Disable environment type since staging doesn't exist yet 5 | # echo -n "Would you like to connect to staging or production? [production] " 6 | # read dep_type 7 | 8 | # if [ -z \$dep_type ] || [ \$dep_type == "production" ] || [ \$dep_type == "prod" ]; then 9 | # namespace="default" 10 | # elif [ \$dep_type == "staging" ]; then 11 | # namespace="staging" 12 | # else 13 | # echo "Please enter nothing, production, prod, or staging. You entered: \${dep_type}" 14 | # echo "Press enter to exit" 15 | # read dummy 16 | # exit 1 17 | # fi 18 | namespace="default" 19 | 20 | echo "List of deployments: " 21 | kubectl get deployment -n \$namespace 22 | 23 | echo -n "Enter deployment name: " 24 | read dep_name 25 | 26 | kubectl exec -it -n \$namespace \$(kubectl get pod -n \$namespace | grep \$dep_name | grep Running | head -n 1 | cut -d " " -f 1) -- /bin/bash 27 | echo "Press enter to exit" 28 | read 29 | -------------------------------------------------------------------------------- /terraform/files/bastion/container_exec_entry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Need to escape all $ because of TF formatting 4 | if [[ \$2 == "startexec" ]]; then 5 | container_exec.sh 6 | exit \$? 7 | fi 8 | 9 | echo "List of active sessions:" 10 | 11 | tmux ls 2>/dev/null || echo "No active sessions" 12 | 13 | echo -n "Enter session name: " 14 | 15 | read session_name 16 | 17 | tmux has-session -t \$session_name 2>/dev/null 18 | 19 | 20 | if [[ \$? != 0 ]]; then 21 | tmux new -s \$session_name "startexec" 22 | else 23 | tmux attach -t \$session_name 24 | fi 25 | -------------------------------------------------------------------------------- /terraform/files/bastion/user_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Variables 4 | KUBECTL_VERSION="v1.20.4" 5 | AWS_CLI_VERSION="2.0.30" 6 | 7 | # Install packages 8 | apt-get install -y unzip 9 | 10 | # Move exec files to local bin 11 | cat < /usr/local/bin/container_exec_entry.sh 12 | ${CONTAIN_EXEC_ENTRY} 13 | EOF 14 | cat < /usr/local/bin/container_exec.sh 15 | ${CONTAIN_EXEC} 16 | EOF 17 | chmod +x /usr/local/bin/container_exec_entry.sh 18 | chmod +x /usr/local/bin/container_exec.sh 19 | 20 | # Make user 21 | adduser --disabled-password --gecos "" jump 22 | 23 | # Add SSH keys to jump user 24 | mkdir /home/jump/.ssh 25 | cat < /home/jump/.ssh/authorized_keys 26 | ${SSH_AUTHORIZED_KEYS} 27 | EOF 28 | chown jump:jump /home/jump/.ssh/authorized_keys 29 | 30 | # Install kubectl 31 | curl -LO "https://dl.k8s.io/release/$${KUBECTL_VERSION}/bin/linux/amd64/kubectl" 32 | chmod +x kubectl 33 | mv kubectl /usr/local/bin/ 34 | 35 | # Install AWS CLI 36 | curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-$${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" 37 | unzip awscliv2.zip 38 | ./aws/install 39 | rm awscliv2.zip 40 | rm -r aws 41 | 42 | # Generate kubeconfig 43 | su jump -c "aws eks --region us-east-1 update-kubeconfig --name production" 44 | 45 | # Set jump shell 46 | chsh jump -s /usr/local/bin/container_exec_entry.sh 47 | -------------------------------------------------------------------------------- /terraform/files/kubeconfig: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | clusters: 3 | - cluster: 4 | server: ${endpoint} 5 | certificate-authority-data: ${ca} 6 | name: production 7 | contexts: 8 | - context: 9 | cluster: production 10 | user: aws 11 | name: production 12 | current-context: aws 13 | kind: Config 14 | preferences: {} 15 | users: 16 | - name: aws 17 | user: 18 | exec: 19 | apiVersion: client.authentication.k8s.io/v1alpha1 20 | command: aws 21 | args: 22 | - "eks" 23 | - "get-token" 24 | - "--cluster-name" 25 | - "production" 26 | - "--role" 27 | - "${role}" 28 | -------------------------------------------------------------------------------- /terraform/files/vault_user_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat < /etc/vault.d/vault.hcl 4 | ui = true 5 | 6 | listener "tcp" { 7 | address = "0.0.0.0:8200" 8 | tls_cert_file = "/opt/vault/tls/tls.crt" 9 | tls_key_file = "/opt/vault/tls/tls.key" 10 | } 11 | 12 | storage "postgresql" { 13 | connection_url = "${connection_url}" 14 | max_parallel = "4" 15 | } 16 | 17 | seal "awskms" { 18 | region = "us-east-1" 19 | kms_key_id = "${kms_key_id}" 20 | } 21 | EOF 22 | -------------------------------------------------------------------------------- /terraform/gh-actions.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_user" "gh-actions" { 2 | name = "gh-actions" 3 | 4 | tags = { 5 | created-by = "terraform" 6 | } 7 | } 8 | 9 | resource "aws_iam_access_key" "gh-actions" { 10 | user = aws_iam_user.gh-actions.name 11 | } 12 | 13 | resource "aws_iam_user_policy" "gh-actions-assume-kubectl" { 14 | name = "kubectl" 15 | user = aws_iam_user.gh-actions.name 16 | policy = data.aws_iam_policy_document.assume-kubectl.json 17 | } 18 | 19 | resource "aws_iam_user_policy" "gh-actions-view-k8s" { 20 | name = "view-eks" 21 | user = aws_iam_user.gh-actions.name 22 | policy = data.aws_iam_policy_document.view-k8s.json 23 | } 24 | -------------------------------------------------------------------------------- /terraform/github.tf: -------------------------------------------------------------------------------- 1 | resource "github_actions_organization_secret" "aws_account_id" { 2 | secret_name = "AWS_ACCOUNT_ID" 3 | visibility = "all" 4 | plaintext_value = data.aws_caller_identity.current.account_id 5 | } 6 | 7 | resource "github_actions_organization_secret" "aws_access_key" { 8 | secret_name = "GH_AWS_ACCESS_KEY_ID" 9 | visibility = "all" 10 | plaintext_value = aws_iam_access_key.gh-actions.id 11 | } 12 | 13 | resource "github_actions_organization_secret" "aws_secret_key" { 14 | secret_name = "GH_AWS_SECRET_ACCESS_KEY" 15 | visibility = "all" 16 | plaintext_value = aws_iam_access_key.gh-actions.secret 17 | } 18 | 19 | // Set github action secrets for private repositories (bc no access to org secrets) 20 | data "github_repositories" "private_repos" { 21 | query = "org:pennlabs is:private" 22 | } 23 | 24 | resource "github_actions_secret" "private_repos_aws_account_id" { 25 | for_each = toset(data.github_repositories.private_repos.names) 26 | repository = each.key 27 | secret_name = "AWS_ACCOUNT_ID" 28 | plaintext_value = data.aws_caller_identity.current.account_id 29 | } 30 | 31 | resource "github_actions_secret" "private_repos_aws_access_key" { 32 | for_each = toset(data.github_repositories.private_repos.names) 33 | repository = each.key 34 | secret_name = "GH_AWS_ACCESS_KEY_ID" 35 | plaintext_value = aws_iam_access_key.gh-actions.id 36 | } 37 | 38 | resource "github_actions_secret" "private_repos_aws_secret_key" { 39 | for_each = toset(data.github_repositories.private_repos.names) 40 | repository = each.key 41 | secret_name = "GH_AWS_SECRET_ACCESS_KEY" 42 | plaintext_value = aws_iam_access_key.gh-actions.secret 43 | } 44 | -------------------------------------------------------------------------------- /terraform/helm/aws-node-termination-handler.yaml: -------------------------------------------------------------------------------- 1 | ## enableSpotInterruptionDraining If true, drain nodes when the spot interruption termination notice is received 2 | enableSpotInterruptionDraining: "true" 3 | 4 | # nodeSelector tells both linux and windows daemonsets where to place the node-termination-handler 5 | # pods. By default, this value is empty and every node will receive a pod. 6 | nodeSelector: 7 | eks.amazonaws.com/capacityType: SPOT 8 | 9 | enablePrometheusServer: true 10 | 11 | podAnnotations: 12 | prometheus.io/scrape: "true" 13 | prometheus.io/port: "9092" 14 | 15 | image: 16 | pullSecrets: 17 | - docker-pull-secret 18 | -------------------------------------------------------------------------------- /terraform/helm/bitwarden.yaml: -------------------------------------------------------------------------------- 1 | applications: 2 | - name: bitwarden 3 | image: bitwardenrs/server 4 | tag: "1.19.0" 5 | secret: bitwarden 6 | ingress: 7 | hosts: 8 | - host: bitwarden.pennlabs.org 9 | paths: ["/"] 10 | extraEnv: 11 | - name: SIGNUPS_ALLOWED 12 | value: "false" 13 | - name: ENABLE_DB_WAL 14 | value: "false" 15 | -------------------------------------------------------------------------------- /terraform/helm/cert-manager.yaml: -------------------------------------------------------------------------------- 1 | installCRDs: true 2 | 3 | serviceAccount: 4 | annotations: 5 | eks.amazonaws.com/role-arn: arn:aws:iam::449445102765:role/cert-manager 6 | 7 | #securityContext: 8 | # enabled: true 9 | # fsGroup: 1001 10 | 11 | global: 12 | imagePullSecrets: 13 | - name: docker-pull-secret 14 | -------------------------------------------------------------------------------- /terraform/helm/datadog.yaml: -------------------------------------------------------------------------------- 1 | datadog: 2 | apiKeyExistingSecret: datadog 3 | logs: 4 | enabled: true 5 | containerCollectAll: true 6 | 7 | confd: 8 | cert_manager.yaml: |- 9 | ad_identifiers: 10 | - cert-manager 11 | init_config: 12 | instances: 13 | - prometheus_url: http://%%host%%:9402/metrics 14 | postgres.yaml: |- 15 | init_config: 16 | instances: 17 | - host: "%%env_POSTGRES_HOST%%" 18 | port: 5432 19 | username: datadog 20 | password: "%%env_POSTGRES_PASSWORD%%" 21 | 22 | agents: 23 | image: 24 | repository: pennlabs/datadog-agent 25 | tag: 7479c2dd283d4ecc808fc65107048658dde778a2 26 | doNotCheckTag: true 27 | containers: 28 | agent: 29 | envFrom: 30 | - secretRef: 31 | name: datadog 32 | -------------------------------------------------------------------------------- /terraform/helm/db-backup.yaml: -------------------------------------------------------------------------------- 1 | cronjobs: 2 | - name: db-backup 3 | schedule: "21 2 * * *" 4 | secret: db-backup 5 | image: pennlabs/pg-s3-backup 6 | tag: 18806efdf96777fce2341b8eb81c95bf1a7d6897 7 | extraEnv: 8 | - name: AWS_DEFAULT_REGION 9 | value: "us-east-1" 10 | 11 | rbac: 12 | createSA: true 13 | roleARN: ${roleARN} 14 | -------------------------------------------------------------------------------- /terraform/helm/grafana.yaml: -------------------------------------------------------------------------------- 1 | ingress: 2 | enabled: true 3 | hosts: 4 | - grafana.pennlabs.org 5 | tls: 6 | - secretName: pennlabs-org-tls 7 | hosts: 8 | - grafana.pennlabs.org 9 | 10 | admin: 11 | existingSecret: grafana 12 | userKey: ADMIN_USER 13 | passwordKey: ADMIN_PASSWORD 14 | 15 | persistence: 16 | enabled: true 17 | type: statefulset 18 | size: 10Gi 19 | 20 | plugins: 21 | - grafana-piechart-panel 22 | 23 | notifiers: 24 | notifiers.yaml: 25 | notifiers: 26 | - name: Slack 27 | type: slack 28 | uid: slack 29 | org_id: 1 30 | is_default: true 31 | send_reminder: false 32 | settings: 33 | url: ${SLACK_NOTIFICATION_URL} 34 | 35 | dashboardProviders: 36 | dashboardproviders.yaml: 37 | apiVersion: 1 38 | providers: 39 | - name: "default" 40 | orgId: 1 41 | folder: "" 42 | type: file 43 | disableDeletion: false 44 | editable: true 45 | options: 46 | path: /var/lib/grafana/dashboards/default 47 | 48 | dashboards: 49 | default: 50 | node-exporter: 51 | gnetId: 1860 52 | revision: 19 53 | cert-manager: 54 | url: https://raw.githubusercontent.com/pennlabs/infrastructure/master/grafana-dashboards/cert-manager.json 55 | traefik: 56 | url: https://raw.githubusercontent.com/pennlabs/infrastructure/master/grafana-dashboards/traefik.json 57 | pod-alerting-dashboard: 58 | url: https://raw.githubusercontent.com/pennlabs/infrastructure/master/grafana-dashboards/pod-alerting-dashboard.json 59 | pod-dashboard: 60 | url: https://raw.githubusercontent.com/pennlabs/infrastructure/master/grafana-dashboards/pod-dashboard.json 61 | 62 | datasources: 63 | datasources.yaml: 64 | apiVersion: 1 65 | datasources: 66 | - name: Prometheus 67 | type: prometheus 68 | url: http://prometheus-server.monitoring 69 | access: proxy 70 | orgId: 1 71 | 72 | grafana.ini: 73 | server: 74 | domain: "grafana.pennlabs.org" 75 | root_url: "https://%(domain)s/" 76 | auth.github: 77 | enabled: true 78 | scopes: user:email,read:org 79 | auth_url: https://github.com/login/oauth/authorize 80 | token_url: https://github.com/login/oauth/access_token 81 | api_url: https://api.github.com/user 82 | allow_sign_up: true 83 | allowed_organizations: pennlabs 84 | 85 | envFromSecret: "grafana" 86 | 87 | image: 88 | pullSecrets: 89 | - docker-pull-secret 90 | -------------------------------------------------------------------------------- /terraform/helm/ingress/bitwarden-ingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | annotations: 5 | name: bitwarden-bitwarden 6 | spec: 7 | routes: 8 | - kind: Rule 9 | match: Host(`bitwarden.pennlabs.org`) && PathPrefix(`/`) 10 | services: 11 | - kind: Service 12 | name: bitwarden-bitwarden 13 | namespace: default 14 | port: 80 15 | tls: 16 | domains: 17 | - main: pennlabs.org 18 | secretName: pennlabs-org-tls -------------------------------------------------------------------------------- /terraform/helm/ingress/expander-expander.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | annotations: 5 | name: expander-expander 6 | spec: 7 | routes: 8 | - kind: Rule 9 | match: Host(`u.pennlabs.org`) && PathPrefix(`/`) 10 | services: 11 | - kind: Service 12 | name: expander-expander 13 | namespace: default 14 | port: 80 15 | tls: 16 | domains: 17 | - main: pennlabs.org 18 | secretName: pennlabs-org-tls -------------------------------------------------------------------------------- /terraform/helm/ingress/penn-basics-react.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | annotations: 5 | name: penn-basics-react 6 | spec: 7 | routes: 8 | - kind: Rule 9 | match: Host(`pennbasics.com`) && PathPrefix(`/`) 10 | services: 11 | - kind: Service 12 | name: penn-basics-react 13 | passHostHeader: true 14 | port: 80 15 | tls: 16 | domains: 17 | - main: pennbasics.com 18 | secretName: pennbasics-com-tls -------------------------------------------------------------------------------- /terraform/helm/ingress/penn-clubs-hub-django-wsgi.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | annotations: 5 | name: penn-clubs-hub-django-wsgi 6 | spec: 7 | entryPoints: 8 | - websecure 9 | routes: 10 | - kind: Rule 11 | match: Host(`hub.provost.upenn.edu`) && PathPrefix(`/api`) 12 | services: 13 | - kind: Service 14 | name: penn-clubs-hub-django-wsgi 15 | passHostHeader: true 16 | port: 80 17 | tls: 18 | secretName: hub-provost-upenn-edu-tls -------------------------------------------------------------------------------- /terraform/helm/ingress/penn-clubs-hub-react.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | annotations: 5 | name: penn-clubs-hub-react 6 | spec: 7 | entryPoints: 8 | - websecure 9 | routes: 10 | - kind: Rule 11 | match: Host(`hub.provost.upenn.edu`) && PathPrefix(`/`)) 12 | services: 13 | - kind: Service 14 | name: penn-clubs-hub-react 15 | passHostHeader: true 16 | port: 80 17 | tls: 18 | secretName: hub-provost-upenn-edu-tls -------------------------------------------------------------------------------- /terraform/helm/ingress/platform-dev.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | annotations: 5 | name: platform-dev 6 | spec: 7 | routes: 8 | - kind: Rule 9 | match: Host(`platform-dev.pennlabs.org`) && PathPrefix(`/`) 10 | services: 11 | - kind: Service 12 | name: platform-dev 13 | namespace: default 14 | port: 8080 15 | tls: 16 | domains: 17 | - main: pennlabs.org 18 | secretName: pennlabs-org-tls -------------------------------------------------------------------------------- /terraform/helm/prometheus.yaml: -------------------------------------------------------------------------------- 1 | alertmanager: 2 | # we're using grafana alerts for the time being 3 | enabled: false 4 | 5 | server: 6 | image: 7 | repository: prom/prometheus 8 | tag: v2.13.1 9 | pullPolicy: IfNotPresent 10 | persistentVolume: 11 | enabled: true 12 | accessModes: 13 | - ReadWriteOnce 14 | size: 8Gi 15 | 16 | imagePullSecrets: 17 | - name: docker-pull-secret 18 | -------------------------------------------------------------------------------- /terraform/helm/team-sync.yaml: -------------------------------------------------------------------------------- 1 | cronjobs: 2 | - name: team-sync 3 | schedule: "*/10 * * * *" 4 | secret: team-sync 5 | image: pennlabs/team-sync 6 | tag: d6cdea59a93adb81d26053049c82ae9f5f5779ae 7 | failureLimit: 1 8 | extraEnv: 9 | - name: VAULT_ADDR 10 | value: https://vault.pennlabs.org 11 | - name: PLATFORM_URL 12 | value: https://platform.pennlabs.org 13 | 14 | rbac: 15 | createSA: true 16 | roleARN: ${roleARN} 17 | -------------------------------------------------------------------------------- /terraform/helm/traefik.yaml: -------------------------------------------------------------------------------- 1 | deployment: 2 | replicas: 2 3 | imagePullSecrets: 4 | - name: dockerPullSecret 5 | podAnnotations: 6 | ad.datadoghq.com/traefik.check_names: | 7 | ["openmetrics"] 8 | ad.datadoghq.com/traefik.init_configs: | 9 | [{}] 10 | ad.datadoghq.com/traefik.instances: | 11 | [ 12 | { 13 | "openmetrics_endpoint": "http://%%host%%:9100/metrics", 14 | "namespace": "traefik", 15 | "metrics": [".*"], 16 | "max_returned_metrics": 3000 17 | } 18 | ] 19 | 20 | logs: 21 | general: 22 | access: 23 | enabled: true -------------------------------------------------------------------------------- /terraform/helm/vault-secret-sync.yaml: -------------------------------------------------------------------------------- 1 | namespaces: 2 | - default 3 | - monitoring 4 | # - staging 5 | 6 | role_arn: ${role_arn} 7 | -------------------------------------------------------------------------------- /terraform/iam.tf: -------------------------------------------------------------------------------- 1 | module "iam-products" { 2 | for_each = local.iam_service_accounts 3 | source = "./modules/iam" 4 | role = each.key 5 | oidc_issuer_url = module.eks-production.cluster_oidc_issuer_url 6 | oidc_provider_arn = module.eks-production.oidc_provider_arn 7 | } 8 | 9 | module "iam-secret-sync" { 10 | source = "./modules/iam" 11 | role = "secret-sync" 12 | namespaces = ["default", "monitoring"] 13 | oidc_issuer_url = module.eks-production.cluster_oidc_issuer_url 14 | oidc_provider_arn = module.eks-production.oidc_provider_arn 15 | } 16 | 17 | module "iam-cert-manager" { 18 | source = "./modules/iam" 19 | role = "cert-manager" 20 | namespaces = ["cert-manager"] 21 | oidc_issuer_url = module.eks-production.cluster_oidc_issuer_url 22 | oidc_provider_arn = module.eks-production.oidc_provider_arn 23 | } 24 | -------------------------------------------------------------------------------- /terraform/modules/README.md: -------------------------------------------------------------------------------- 1 | # Modules 2 | 3 | Reusable terraform modules to keep our infrastructure DRY. 4 | 5 | We've created the following modules: 6 | 7 | * [Base Cluster](base_cluster) - a barebones K8s cluster with additional software installed 8 | * [IAM](iam) - a module to create an IAM role that can be assumed from Kubernetes 9 | * [Vault](vault) - a module to configure vault with all the secrets we need 10 | * [Vault Flush](vault_flush) - a module to flush updated secrets to vault 11 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/README.md: -------------------------------------------------------------------------------- 1 | # Base Cluster 2 | 3 | A terraform module to populate a Kubernetes cluster with some useful additional software. 4 | 5 | ## Inputs 6 | 7 | | Name | Description | 8 | | ------------------------ | ----------------------------------------------------- | 9 | | traefik_values | Values to provide to the traefik helm chart | 10 | | vault_secret_sync_values | Values to provide to the Vault Secret Sync helm chart | 11 | | prometheus_values | Values to provide to the Prometheus helm chart | 12 | 13 | ## main.tf 14 | 15 | Doesn't do anything 16 | 17 | ## cert-manager.tf 18 | 19 | Configures cert-manager by 20 | 21 | * Creating the `cert-manager` namespace 22 | * Installing the `cert-manager` helm chart to the `cert-manager` namespace 23 | * Creating a [ClusterIssuer](https://cert-manager.io/docs/concepts/issuer/) to issue TLS certificates from [Lets Encrypt](https://letsencrypt.org/) using the DNS01 challenge with [delegated domains](https://cert-manager.io/docs/configuration/acme/dns01/#delegated-domains-for-dns01) through our Cloudflare account. 24 | * Creating a wildcard certificate for `*.pennlabs.org` in the `default` namespace 25 | 26 | ## monitoring.tf 27 | 28 | Configure our monitoring stack by 29 | 30 | * Creating the `monitoring` namespace 31 | * Installing the [prometheus helm chart](https://github.com/helm/charts/tree/master/stable/prometheus) with the inputted values 32 | 33 | ## traefik.tf 34 | 35 | Installs the [traefik 1.7 helm chart](https://github.com/helm/charts/tree/master/stable/traefik) with the inputted values 36 | 37 | ## vault-secret-sync.tf 38 | 39 | Installs the [vault secret sync helm chart](https://github.com/pennlabs/vault-secret-sync/) with the inputted values 40 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/cert-manager.tf: -------------------------------------------------------------------------------- 1 | resource "kubernetes_namespace" "cert-manager" { 2 | metadata { 3 | name = "cert-manager" 4 | } 5 | } 6 | 7 | resource "helm_release" "cert-manager" { 8 | name = "cert-manager" 9 | repository = "https://charts.jetstack.io" 10 | chart = "cert-manager" 11 | version = "v1.11.0" 12 | namespace = kubernetes_namespace.cert-manager.metadata[0].name 13 | // This is set to ensure that cert-manager is working before the CRs are applied 14 | atomic = true 15 | values = var.cert_manager_values 16 | } 17 | 18 | resource "time_sleep" "cert-manager-cr" { 19 | // Used to allow cert-manager time to initialize 20 | depends_on = [helm_release.cert-manager] 21 | create_duration = "1m" 22 | } 23 | 24 | resource "helm_release" "labs-clusterissuer" { 25 | name = "labs-clusterissuer" 26 | repository = "https://helm.pennlabs.org" 27 | chart = "helm-wrapper" 28 | version = "0.1.0" 29 | values = [file("${path.module}/clusterissuer.yaml")] 30 | 31 | depends_on = [ 32 | time_sleep.cert-manager-cr 33 | ] 34 | } 35 | 36 | resource "helm_release" "pennlabs-wildcard-cert" { 37 | name = "pennlabs-wildcard-cert" 38 | repository = "https://helm.pennlabs.org" 39 | chart = "helm-wrapper" 40 | version = "0.1.0" 41 | values = [file("${path.module}/wildcard-cert.yaml")] 42 | 43 | depends_on = [ 44 | time_sleep.cert-manager-cr 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/clusterissuer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: ClusterIssuer 3 | metadata: 4 | name: wildcard-letsencrypt-prod 5 | spec: 6 | acme: 7 | server: https://acme-v02.api.letsencrypt.org/directory 8 | email: letsencrypt@pennlabs.org 9 | privateKeySecretRef: 10 | name: letsencrypt 11 | solvers: 12 | - selector: {} 13 | dns01: 14 | cnameStrategy: Follow 15 | route53: 16 | region: us-east-1 17 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/datadog.tf: -------------------------------------------------------------------------------- 1 | resource "helm_release" "datadog" { 2 | name = "datadog" 3 | repository = "https://helm.datadoghq.com" 4 | chart = "datadog" 5 | namespace = kubernetes_namespace.monitoring.metadata[0].name 6 | 7 | values = var.datadog_values 8 | } 9 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/main.tf: -------------------------------------------------------------------------------- 1 | // Nothing here 2 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/monitoring.tf: -------------------------------------------------------------------------------- 1 | resource "kubernetes_namespace" "monitoring" { 2 | metadata { 3 | name = "monitoring" 4 | } 5 | } 6 | 7 | resource "helm_release" "prometheus" { 8 | name = "prometheus" 9 | repository = "https://charts.helm.sh/stable" 10 | chart = "prometheus" 11 | version = "11.2.3" 12 | namespace = kubernetes_namespace.monitoring.metadata[0].name 13 | 14 | values = var.prometheus_values 15 | } 16 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/redis.tf: -------------------------------------------------------------------------------- 1 | resource "kubernetes_config_map" "redis_config_map" { 2 | metadata { 3 | name = "redis-config" 4 | } 5 | 6 | data = { 7 | "redis-config" = <<-EOF 8 | save 3600 30 9 | dir /redis-master-data/ 10 | dbfilename dump.rdb 11 | protected-mode no 12 | EOF 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/traefik.tf: -------------------------------------------------------------------------------- 1 | resource "helm_release" "traefik" { 2 | name = "traefik" 3 | repository = "https://traefik.github.io/charts" 4 | chart = "traefik" 5 | version = "21.0.0" 6 | namespace = "kube-system" 7 | 8 | values = var.traefik_values 9 | } 10 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/variables.tf: -------------------------------------------------------------------------------- 1 | // Traefik values 2 | variable "traefik_values" { 3 | description = "Values to provide to the Traefik helm chart" 4 | type = list(string) 5 | } 6 | 7 | // Vault Secret Sync values 8 | variable "vault_secret_sync_values" { 9 | description = "Values to provide to the Vault Secret Sync helm chart" 10 | type = list(string) 11 | } 12 | 13 | // Prometheus values 14 | variable "prometheus_values" { 15 | description = "Values to provide to the Prometheus helm chart" 16 | type = list(string) 17 | } 18 | 19 | // Cert Manager values 20 | variable "cert_manager_values" { 21 | description = "Values to provide to the Cert Manager helm chart" 22 | type = list(string) 23 | } 24 | 25 | // Datadog values 26 | variable "datadog_values" { 27 | description = "Values to provide to the Datadog helm chart" 28 | type = list(string) 29 | } 30 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/vault-secret-sync.tf: -------------------------------------------------------------------------------- 1 | resource "helm_release" "vault-secret-sync" { 2 | name = "vault-secret-sync" 3 | repository = "https://helm.pennlabs.org" 4 | chart = "vault-secret-sync" 5 | version = "0.1.5" 6 | values = var.vault_secret_sync_values 7 | depends_on = [ 8 | kubernetes_namespace.monitoring, 9 | kubernetes_namespace.cert-manager, 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /terraform/modules/base_cluster/wildcard-cert.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: Certificate 3 | metadata: 4 | annotations: 5 | meta.helm.sh/release-name: pennlabs-wildcard-cert 6 | meta.helm.sh/release-namespace: default 7 | name: pennlabs-org 8 | spec: 9 | secretName: pennlabs-org-tls 10 | dnsNames: 11 | - "pennlabs.org" 12 | - "*.pennlabs.org" 13 | issuerRef: 14 | name: wildcard-letsencrypt-prod 15 | kind: ClusterIssuer 16 | group: cert-manager.io 17 | -------------------------------------------------------------------------------- /terraform/modules/domain/README.md: -------------------------------------------------------------------------------- 1 | # Domain 2 | 3 | A terraform module to configure basic DNS records for a Penn Labs domain that: 4 | 5 | * Point the apex domain to traefik 6 | * CNAME all subdomains to the apex domain 7 | * Create SPF, DKIM, and CNAME records to send mail through mailgun 8 | * Configure MX records to receive mail through Google. 9 | 10 | | Name | Description | 11 | | --------------- | ------------------------------------- | 12 | | domain | Domain name to configure | 13 | | traefik_lb_name | DNS name of the traefik load balancer | 14 | | traefik_zone_id | Zone ID of the traefik load balancer | 15 | 16 | ## Outputs 17 | 18 | None 19 | -------------------------------------------------------------------------------- /terraform/modules/domain/main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_route53_zone" "domain" { 2 | name = var.domain 3 | } 4 | 5 | resource "aws_route53_record" "apex-domain" { 6 | zone_id = aws_route53_zone.domain.zone_id 7 | name = "" 8 | type = "A" 9 | 10 | alias { 11 | name = var.traefik_lb_name 12 | zone_id = var.traefik_zone_id 13 | evaluate_target_health = false 14 | } 15 | } 16 | 17 | resource "aws_route53_record" "wildcard" { 18 | zone_id = aws_route53_zone.domain.zone_id 19 | name = "*" 20 | type = "CNAME" 21 | ttl = 3600 22 | records = [aws_route53_zone.domain.name] 23 | } 24 | 25 | resource "aws_route53_record" "spf" { 26 | zone_id = aws_route53_zone.domain.zone_id 27 | name = "" 28 | type = "TXT" 29 | ttl = 3600 30 | records = ["v=spf1 include:mailgun.org ~all"] 31 | multivalue_answer_routing_policy = true 32 | set_identifier = "spf" 33 | } 34 | 35 | resource "aws_route53_record" "mailgun" { 36 | zone_id = aws_route53_zone.domain.zone_id 37 | name = "email" 38 | type = "CNAME" 39 | ttl = 3600 40 | records = ["mailgun.org."] 41 | } 42 | 43 | resource "aws_route53_record" "gmail" { 44 | zone_id = aws_route53_zone.domain.zone_id 45 | name = "" 46 | type = "MX" 47 | ttl = 3600 48 | records = [ 49 | "5 gmr-smtp-in.l.google.com.", 50 | "10 alt1.gmr-smtp-in.l.google.com.", 51 | "20 alt2.gmr-smtp-in.l.google.com.", 52 | "30 alt3.gmr-smtp-in.l.google.com.", 53 | "40 alt4.gmr-smtp-in.l.google.com.", 54 | ] 55 | } 56 | -------------------------------------------------------------------------------- /terraform/modules/domain/outputs.tf: -------------------------------------------------------------------------------- 1 | output "zone_id" { 2 | value = aws_route53_zone.domain.zone_id 3 | } 4 | -------------------------------------------------------------------------------- /terraform/modules/domain/variables.tf: -------------------------------------------------------------------------------- 1 | variable "domain" { 2 | description = "Domain name" 3 | type = string 4 | } 5 | 6 | variable "traefik_lb_name" { 7 | description = "DNS name of the traefik load balancer" 8 | type = string 9 | } 10 | 11 | variable "traefik_zone_id" { 12 | description = "Zone ID of the traefik load balancer" 13 | type = string 14 | } 15 | -------------------------------------------------------------------------------- /terraform/modules/iam/README.md: -------------------------------------------------------------------------------- 1 | # IAM 2 | 3 | A terraform module to create an IAM role that can be assumed by a Kubernetes Service Account with the same name. 4 | 5 | ## Inputs 6 | 7 | | Name | Description | 8 | | ----------------- | --------------------------------------- | 9 | | role | Name of K8s SA (and generated IAM role) | 10 | | namespaces | Namespaces of the K8s SA | 11 | | oidc_issuer_url | URL of the K8s oidc issuer | 12 | | oidc_provider_arn | ARN of the K8s oidc issuer | 13 | 14 | ## Outputs 15 | 16 | None 17 | -------------------------------------------------------------------------------- /terraform/modules/iam/main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_role" "role" { 2 | assume_role_policy = data.aws_iam_policy_document.k8s.json 3 | name = var.role 4 | tags = { 5 | created-by = "terraform" 6 | } 7 | } 8 | 9 | data "aws_iam_policy_document" "k8s" { 10 | statement { 11 | actions = ["sts:AssumeRoleWithWebIdentity"] 12 | effect = "Allow" 13 | 14 | condition { 15 | test = "StringEquals" 16 | variable = "${replace(var.oidc_issuer_url, "https://", "")}:sub" 17 | values = [for namespace in var.namespaces : "system:serviceaccount:${namespace}:${var.role}"] 18 | } 19 | 20 | principals { 21 | identifiers = [var.oidc_provider_arn] 22 | type = "Federated" 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /terraform/modules/iam/outputs.tf: -------------------------------------------------------------------------------- 1 | output "role-id" { 2 | value = aws_iam_role.role.id 3 | } 4 | 5 | output "role-arn" { 6 | value = aws_iam_role.role.arn 7 | } 8 | -------------------------------------------------------------------------------- /terraform/modules/iam/variables.tf: -------------------------------------------------------------------------------- 1 | variable "role" { 2 | description = "Name of K8s SA (and generated IAM role)" 3 | type = string 4 | } 5 | 6 | variable "namespaces" { 7 | description = "Namespace(s) of the k8s SA" 8 | type = set(string) 9 | default = ["default"] 10 | } 11 | 12 | variable "oidc_issuer_url" { 13 | description = "URL of the K8s oidc issuer" 14 | type = string 15 | } 16 | 17 | variable "oidc_provider_arn" { 18 | description = "ARN of the K8s oidc issuer" 19 | type = string 20 | } 21 | -------------------------------------------------------------------------------- /terraform/modules/vault/grafana.tf: -------------------------------------------------------------------------------- 1 | resource "random_password" "grafana-admin" { 2 | length = 64 3 | special = false 4 | } 5 | 6 | resource "vault_generic_secret" "grafana" { 7 | path = "${vault_mount.secrets.path}/production/default/grafana" 8 | 9 | data_json = jsonencode({ 10 | "ADMIN_USER" = "admin" 11 | "ADMIN_PASSWORD" = random_password.grafana-admin.result 12 | "GF_AUTH_GITHUB_CLIENT_ID" = var.GF_GH_CLIENT_ID 13 | "GF_AUTH_GITHUB_CLIENT_SECRET" = var.GF_GH_CLIENT_SECRET 14 | "SLACK_NOTIFICATION_URL" = var.GF_SLACK_URL 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /terraform/modules/vault/main.tf: -------------------------------------------------------------------------------- 1 | resource "vault_mount" "secrets" { 2 | path = "secrets" 3 | type = "kv-v2" 4 | description = "Secrets backend" 5 | } 6 | 7 | resource "vault_auth_backend" "aws" { 8 | type = "aws" 9 | } 10 | 11 | resource "vault_github_auth_backend" "pennlabs" { 12 | organization = "pennlabs" 13 | } 14 | 15 | resource "vault_github_team" "sre" { 16 | backend = vault_github_auth_backend.pennlabs.id 17 | team = "sre" 18 | policies = [vault_policy.admin.name] 19 | } 20 | 21 | resource "vault_policy" "admin" { 22 | name = "admin" 23 | policy = templatefile("${path.module}/policies/admin.hcl", { 24 | PATH = vault_mount.secrets.path 25 | } 26 | ) 27 | } 28 | -------------------------------------------------------------------------------- /terraform/modules/vault/outputs.tf: -------------------------------------------------------------------------------- 1 | output "secrets-path" { 2 | value = vault_mount.secrets.path 3 | } 4 | -------------------------------------------------------------------------------- /terraform/modules/vault/policies/admin.hcl: -------------------------------------------------------------------------------- 1 | # Manage auth methods broadly across Vault 2 | path "auth/*" 3 | { 4 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 5 | } 6 | 7 | # Create, update, and delete auth methods 8 | path "sys/auth/*" 9 | { 10 | capabilities = ["create", "update", "delete", "sudo"] 11 | } 12 | 13 | # List auth methods 14 | path "sys/auth" 15 | { 16 | capabilities = ["read"] 17 | } 18 | 19 | # List existing policies 20 | path "sys/policies/acl" 21 | { 22 | capabilities = ["list"] 23 | } 24 | 25 | # Create and manage ACL policies 26 | path "sys/policies/acl/*" 27 | { 28 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 29 | } 30 | 31 | # Create and manage all policies 32 | path "sys/policy/*" 33 | { 34 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 35 | } 36 | 37 | # List, create, update, and delete key/value secrets 38 | path "${PATH}/*" 39 | { 40 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 41 | } 42 | 43 | # Manage secret engines 44 | path "sys/mounts/*" 45 | { 46 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 47 | } 48 | 49 | # List existing secret engines. 50 | path "sys/mounts" 51 | { 52 | capabilities = ["read"] 53 | } 54 | 55 | # Read health checks 56 | path "sys/health" 57 | { 58 | capabilities = ["read", "sudo"] 59 | } 60 | -------------------------------------------------------------------------------- /terraform/modules/vault/policies/secret-sync.hcl: -------------------------------------------------------------------------------- 1 | path "${PATH}/*" 2 | { 3 | capabilities = ["list", "read"] 4 | } 5 | 6 | path "sys/mounts" 7 | { 8 | capabilities = ["read"] 9 | } 10 | 11 | path "sys/mounts/*" 12 | { 13 | capabilities = ["list", "read"] 14 | } 15 | -------------------------------------------------------------------------------- /terraform/modules/vault/policies/team-sync.hcl: -------------------------------------------------------------------------------- 1 | # Manage auth methods broadly across Vault 2 | path "auth/*" 3 | { 4 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 5 | } 6 | 7 | # Create, update, and delete auth methods 8 | path "sys/auth/*" 9 | { 10 | capabilities = ["create", "update", "delete", "sudo"] 11 | } 12 | 13 | # List auth methods 14 | path "sys/auth" 15 | { 16 | capabilities = ["read"] 17 | } 18 | 19 | # List existing policies 20 | path "sys/policies/acl" 21 | { 22 | capabilities = ["list"] 23 | } 24 | 25 | # Create and manage ACL policies 26 | path "sys/policies/acl/*" 27 | { 28 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 29 | } 30 | 31 | # Create and manage all policies 32 | path "sys/policy/*" 33 | { 34 | capabilities = ["create", "read", "update", "delete", "list", "sudo"] 35 | } 36 | -------------------------------------------------------------------------------- /terraform/modules/vault/secret-sync.tf: -------------------------------------------------------------------------------- 1 | resource "vault_policy" "secret-sync" { 2 | name = "secret-sync" 3 | policy = templatefile("${path.module}/policies/secret-sync.hcl", { 4 | PATH = vault_mount.secrets.path 5 | } 6 | ) 7 | } 8 | resource "vault_aws_auth_backend_role" "secret-sync" { 9 | role = "secret-sync" 10 | bound_iam_principal_arns = [var.SECRET_SYNC_ARN] 11 | token_policies = [vault_policy.secret-sync.name] 12 | } 13 | -------------------------------------------------------------------------------- /terraform/modules/vault/team-sync.tf: -------------------------------------------------------------------------------- 1 | resource "vault_policy" "team-sync" { 2 | name = "team-sync" 3 | policy = templatefile("${path.module}/policies/team-sync.hcl", { 4 | PATH = vault_mount.secrets.path 5 | } 6 | ) 7 | } 8 | 9 | resource "vault_aws_auth_backend_role" "team-sync" { 10 | role = "team-sync" 11 | bound_iam_principal_arns = [var.TEAM_SYNC_ARN] 12 | token_policies = [vault_policy.team-sync.name] 13 | } 14 | 15 | -------------------------------------------------------------------------------- /terraform/modules/vault/variables.tf: -------------------------------------------------------------------------------- 1 | variable "GF_GH_CLIENT_ID" { 2 | type = string 3 | description = "GitHub Client ID for the Penn Labs Grafana OAuth2 Application" 4 | } 5 | 6 | variable "GF_GH_CLIENT_SECRET" { 7 | type = string 8 | description = "GitHub Client Secret for the Penn Labs Grafana OAuth2 Application" 9 | } 10 | 11 | variable "GF_SLACK_URL" { 12 | type = string 13 | description = "Slack notification URL used for Grafana notifications" 14 | } 15 | 16 | variable "SECRET_SYNC_ARN" { 17 | type = string 18 | description = "Role ARN for secret-sync" 19 | } 20 | 21 | variable "TEAM_SYNC_ARN" { 22 | type = string 23 | description = "Role ARN for team-sync" 24 | } 25 | -------------------------------------------------------------------------------- /terraform/modules/vault_flush/README.md: -------------------------------------------------------------------------------- 1 | # Vault Flush 2 | 3 | A terraform module to flush updated secrets to vault. It takes in an existing secret as well as a map of keys and values to update or create and updates the secret in vault. 4 | 5 | ## Inputs 6 | 7 | | Name | Description | 8 | | ----- | ------------------------------------ | 9 | | path | Path to the existing secret in Vault | 10 | | entry | Entries to replace within the secret | 11 | 12 | Note that a secret must already exist at `path` for this module to work. 13 | 14 | ## Outputs 15 | 16 | None 17 | -------------------------------------------------------------------------------- /terraform/modules/vault_flush/main.tf: -------------------------------------------------------------------------------- 1 | data "vault_generic_secret" "secret" { 2 | path = var.path 3 | } 4 | 5 | resource "vault_generic_secret" "secret" { 6 | path = var.path 7 | data_json = jsonencode(merge(data.vault_generic_secret.secret.data, var.entry)) 8 | } 9 | -------------------------------------------------------------------------------- /terraform/modules/vault_flush/variables.tf: -------------------------------------------------------------------------------- 1 | variable "path" { 2 | description = "Path to the existing secret in Vault" 3 | type = string 4 | } 5 | 6 | variable "entry" { 7 | description = "Entries to replace within the secret" 8 | type = map(any) 9 | } 10 | -------------------------------------------------------------------------------- /terraform/production-cluster.tf: -------------------------------------------------------------------------------- 1 | resource "helm_release" "team-sync" { 2 | name = "team-sync" 3 | repository = "https://helm.pennlabs.org" 4 | chart = "icarus" 5 | version = "0.1.23" 6 | 7 | values = [ 8 | templatefile("helm/team-sync.yaml", { 9 | roleARN = module.iam-products["team-sync"].role-arn 10 | }) 11 | ] 12 | } 13 | 14 | resource "helm_release" "grafana" { 15 | name = "grafana" 16 | repository = "https://charts.helm.sh/stable" 17 | chart = "grafana" 18 | version = "5.1.4" 19 | 20 | values = [file("helm/grafana.yaml")] 21 | } 22 | 23 | resource "helm_release" "bitwarden" { 24 | name = "bitwarden" 25 | repository = "https://helm.pennlabs.org" 26 | chart = "icarus" 27 | version = "0.1.20" 28 | 29 | values = [file("helm/bitwarden.yaml")] 30 | } 31 | 32 | module "production-cluster" { 33 | source = "./modules/base_cluster" 34 | traefik_values = [templatefile("helm/traefik.yaml", { count = local.k8s_cluster_size })] 35 | vault_secret_sync_values = [templatefile("helm/vault-secret-sync.yaml", { 36 | role_arn = module.iam-secret-sync.role-arn 37 | })] 38 | prometheus_values = [file("helm/prometheus.yaml")] 39 | cert_manager_values = [templatefile("helm/cert-manager.yaml", { 40 | roleARN = module.iam-cert-manager.role-arn 41 | })] 42 | datadog_values = [file("helm/datadog.yaml")] 43 | } 44 | 45 | 46 | resource "helm_release" "db-backup" { 47 | name = "db-backup" 48 | repository = "https://helm.pennlabs.org" 49 | chart = "icarus" 50 | version = "0.1.23" 51 | 52 | values = [ 53 | templatefile("helm/db-backup.yaml", { 54 | roleARN = module.iam-products["db-backup"].role-arn 55 | }) 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /terraform/provider.tf: -------------------------------------------------------------------------------- 1 | 2 | // Production K8s cluster 3 | provider "helm" { 4 | kubernetes { 5 | host = data.aws_eks_cluster.production.endpoint 6 | cluster_ca_certificate = base64decode(data.aws_eks_cluster.production.certificate_authority.0.data) 7 | token = data.aws_eks_cluster_auth.production.token 8 | } 9 | } 10 | 11 | provider "kubernetes" { 12 | host = data.aws_eks_cluster.production.endpoint 13 | cluster_ca_certificate = base64decode(data.aws_eks_cluster.production.certificate_authority.0.data) 14 | token = data.aws_eks_cluster_auth.production.token 15 | } 16 | 17 | // Production DB 18 | provider "postgresql" { 19 | host = aws_db_instance.production.address 20 | port = aws_db_instance.production.port 21 | database = "postgres" 22 | expected_version = aws_db_instance.production.engine_version 23 | username = aws_db_instance.production.username 24 | password = aws_db_instance.production.password 25 | superuser = false 26 | sslmode = "require" 27 | } 28 | 29 | provider "github" { 30 | owner = "pennlabs" 31 | } 32 | 33 | terraform { 34 | required_providers { 35 | aws = { 36 | source = "hashicorp/aws" 37 | version = "~> 3.74" 38 | } 39 | random = { 40 | source = "hashicorp/random" 41 | version = "~> 3.1" 42 | } 43 | vault = { 44 | source = "hashicorp/vault" 45 | version = "~> 3.2" 46 | } 47 | time = { 48 | source = "hashicorp/time" 49 | version = "~> 0.7" 50 | } 51 | helm = { 52 | source = "hashicorp/helm" 53 | version = "~> 2.4" 54 | } 55 | kubernetes = { 56 | source = "hashicorp/kubernetes" 57 | version = "~> 2.7" 58 | } 59 | postgresql = { 60 | source = "cyrilgdn/postgresql" 61 | version = "~> 1.15" 62 | } 63 | github = { 64 | source = "integrations/github" 65 | version = "~> 4.20" 66 | } 67 | } 68 | } 69 | 70 | provider "aws" { 71 | region = "us-east-1" 72 | } 73 | 74 | provider "vault" { 75 | address = "https://vault.pennlabs.org" 76 | } 77 | -------------------------------------------------------------------------------- /terraform/route53.tf: -------------------------------------------------------------------------------- 1 | module "domains" { 2 | source = "./modules/domain" 3 | for_each = local.domains 4 | domain = each.key 5 | traefik_lb_name = data.aws_elb.traefik.dns_name 6 | traefik_zone_id = data.aws_elb.traefik.zone_id 7 | } 8 | 9 | data "aws_elb" "traefik" { 10 | name = local.traefik_lb_name 11 | } 12 | 13 | // Bastion 14 | resource "aws_route53_record" "bastion" { 15 | zone_id = module.domains["pennlabs.org"].zone_id 16 | name = "bastion" 17 | type = "CNAME" 18 | ttl = 3600 19 | records = [aws_instance.bastion.public_dns] 20 | } 21 | 22 | 23 | // Cert-Manager IAM 24 | data "aws_iam_policy_document" "cert-manager-iam" { 25 | statement { 26 | actions = [ 27 | "route53:GetChange", 28 | ] 29 | resources = ["arn:aws:route53:::change/*"] 30 | } 31 | statement { 32 | actions = [ 33 | "route53:ChangeResourceRecordSets", 34 | "route53:ListResourceRecordSets", 35 | ] 36 | resources = ["arn:aws:route53:::hostedzone/*"] 37 | } 38 | statement { 39 | actions = [ 40 | "route53:ListHostedZonesByName", 41 | ] 42 | resources = ["*"] 43 | } 44 | } 45 | 46 | resource "aws_iam_role_policy" "cert-manager-iam" { 47 | name = "iam" 48 | role = module.iam-cert-manager.role-id 49 | 50 | policy = data.aws_iam_policy_document.cert-manager-iam.json 51 | } 52 | -------------------------------------------------------------------------------- /terraform/sre.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_user" "sre" { 2 | for_each = local.sre_members 3 | name = each.key 4 | 5 | tags = { 6 | created-by = "terraform" 7 | } 8 | } 9 | 10 | resource "aws_iam_access_key" "sre" { 11 | for_each = local.sre_members 12 | user = aws_iam_user.sre[each.key].name 13 | } 14 | 15 | resource "aws_iam_user_policy" "sre" { 16 | for_each = local.sre_members 17 | name = "kubectl" 18 | user = aws_iam_user.sre[each.key].name 19 | policy = data.aws_iam_policy_document.assume-kubectl.json 20 | } 21 | 22 | resource "vault_generic_secret" "aws-auth" { 23 | for_each = local.sre_members 24 | path = "${module.vault.secrets-path}/breakglass/aws/${each.key}" 25 | 26 | data_json = jsonencode({ 27 | "ACCESS_KEY" = aws_iam_access_key.sre[each.key].id 28 | "SECRET_KEY" = aws_iam_access_key.sre[each.key].secret 29 | }) 30 | } 31 | -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "GH_PERSONAL_TOKEN" { 2 | type = string 3 | description = "GitHub Personal Access token for the Penn Labs Admin account" 4 | } 5 | 6 | variable "GF_GH_CLIENT_ID" { 7 | type = string 8 | description = "GitHub Client ID for the Penn Labs Grafana OAuth2 Application" 9 | } 10 | 11 | variable "GF_GH_CLIENT_SECRET" { 12 | type = string 13 | description = "GitHub Client Secret for the Penn Labs Grafana OAuth2 Application" 14 | } 15 | 16 | variable "GF_SLACK_URL" { 17 | type = string 18 | description = "Slack notification URL used for Grafana notifications" 19 | } 20 | 21 | variable "DOCKERHUB_TOKEN" { 22 | type = string 23 | description = "Read-only API token for Docker Hub" 24 | } 25 | 26 | variable "RDS_PASSWORD" { 27 | type = string 28 | description = "Root password for Production RDS instance" 29 | } -------------------------------------------------------------------------------- /terraform/vpc.tf: -------------------------------------------------------------------------------- 1 | data "aws_availability_zones" "available" {} 2 | 3 | module "vpc" { 4 | // https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/latest 5 | source = "terraform-aws-modules/vpc/aws" 6 | version = "3.11.0" 7 | 8 | name = "vpc" 9 | cidr = "10.0.0.0/16" 10 | azs = data.aws_availability_zones.available.names 11 | # Generate 6 non-overlapping subnets for our VPC. This results in 2^(32-20)=2^12=4096 IPs per subnet. 12 | private_subnets = ["10.0.0.0/20", "10.0.16.0/20", "10.0.32.0/20"] 13 | public_subnets = ["10.0.48.0/20", "10.0.64.0/20", "10.0.80.0/20"] 14 | enable_nat_gateway = true 15 | single_nat_gateway = true 16 | enable_dns_hostnames = true 17 | map_public_ip_on_launch = true 18 | 19 | tags = { 20 | created-by = "terraform" 21 | "kubernetes.io/cluster/${local.k8s_cluster_name}" = "shared" 22 | } 23 | 24 | public_subnet_tags = { 25 | "kubernetes.io/cluster/${local.k8s_cluster_name}" = "shared" 26 | "kubernetes.io/role/elb" = "1" 27 | } 28 | 29 | private_subnet_tags = { 30 | "kubernetes.io/cluster/${local.k8s_cluster_name}" = "shared" 31 | "kubernetes.io/role/internal-elb" = "1" 32 | } 33 | } 34 | --------------------------------------------------------------------------------