├── .github └── workflows │ ├── e2e │ ├── get_astro_env_info │ │ └── action.yaml │ ├── get_bundle_info │ │ └── action.yaml │ ├── get_deployment_info │ │ └── action.yaml │ └── validate_deployment │ │ └── action.yaml │ ├── lint_yaml.yaml │ └── tests.yaml ├── .gitignore ├── .yamllint ├── CODEOWNERS ├── LICENSE ├── README.md ├── action.yaml └── e2e-setup ├── README.md ├── astro-project ├── .astro │ ├── config.yaml │ ├── dag_integrity_exceptions.txt │ └── test_dag_integrity_default.py ├── .dockerignore ├── .gitignore ├── Dockerfile ├── README.md ├── dags │ ├── .airflowignore │ └── exampledag.py ├── packages.txt ├── requirements.txt └── tests │ └── dags │ └── test_dag_example.py ├── dbt └── dbt_project.yml ├── deployment-templates ├── deployment-hibernate.yaml └── deployment.yaml └── mocks ├── astro-deploy.sh ├── dag-deploy-git.sh ├── dbt-deploy-git.sh ├── image-deploy-git.sh └── no-deploy-git.sh /.github/workflows/e2e/get_astro_env_info/action.yaml: -------------------------------------------------------------------------------- 1 | name: Get Astro Environment Info 2 | description: Get information about the Astro environment 3 | inputs: 4 | input_workspace_id: 5 | description: The workspace ID specified via user input 6 | required: true 7 | secret_workspace_id: 8 | description: The workspace ID via pre configured secret 9 | required: true 10 | input_organization_id: 11 | description: The organization ID specified via user input 12 | required: true 13 | secret_organization_id: 14 | description: The organization ID via pre configured secret 15 | required: true 16 | input_astro_api_token: 17 | description: The Astronomer API token specified via user input 18 | required: true 19 | secret_astro_api_token: 20 | description: The Astronomer API token via pre configured secret 21 | required: true 22 | input_astronomer_host: 23 | description: The Astronomer host specified via user input 24 | required: true 25 | secret_astronomer_host: 26 | description: The Astronomer host via pre configured secret 27 | required: true 28 | 29 | outputs: 30 | organization_id: 31 | value: ${{ steps.get-info.outputs.ORGANIZATION_ID }} 32 | workspace_id: 33 | value: ${{ steps.get-info.outputs.WORKSPACE_ID }} 34 | astronomer_host: 35 | value: ${{ steps.get-info.outputs.ASTRONOMER_HOST }} 36 | astro_api_token: 37 | value: ${{ steps.get-info.outputs.ASTRO_API_TOKEN }} 38 | 39 | runs: 40 | using: "composite" 41 | steps: 42 | - name: Get info from inputs or secrets 43 | shell: bash 44 | id: get-info 45 | run: | 46 | if [ "${{ inputs.input_workspace_id }}" != "" ]; then 47 | echo "Using provided workspace_id" 48 | echo "WORKSPACE_ID=${{ inputs.input_workspace_id }}" >> $GITHUB_OUTPUT 49 | else 50 | echo "WORKSPACE_ID=${{ inputs.secret_workspace_id }}" >> $GITHUB_OUTPUT 51 | fi 52 | 53 | if [ "${{ inputs.input_organization_id }}" != "" ]; then 54 | echo "Using provided org_id" 55 | echo "ORGANIZATION_ID=${{ inputs.input_organization_id }}" >> $GITHUB_OUTPUT 56 | else 57 | echo "ORGANIZATION_ID=${{ inputs.secret_organization_id }}" >> $GITHUB_OUTPUT 58 | fi 59 | 60 | if [ "${{ inputs.input_astronomer_host }}" != "" ]; then 61 | echo "Using provided astronomer_host" 62 | echo "ASTRONOMER_HOST=${{ inputs.input_astronomer_host }}" >> $GITHUB_OUTPUT 63 | else 64 | echo "ASTRONOMER_HOST=${{ inputs.secret_astronomer_host }}" >> $GITHUB_OUTPUT 65 | fi 66 | 67 | if [ "${{ inputs.input_astro_api_token }}" != "" ]; then 68 | echo "Using provided token" 69 | echo "ASTRO_API_TOKEN=${{ inputs.input_astro_api_token }}" >> $GITHUB_OUTPUT 70 | echo "ASTRO_API_TOKEN=${{ inputs.input_astro_api_token }}" >> $GITHUB_ENV 71 | else 72 | echo "ASTRO_API_TOKEN=${{ inputs.secret_astro_api_token }}" >> $GITHUB_OUTPUT 73 | echo "ASTRO_API_TOKEN=${{ inputs.secret_astro_api_token }}" >> $GITHUB_ENV 74 | fi 75 | -------------------------------------------------------------------------------- /.github/workflows/e2e/get_bundle_info/action.yaml: -------------------------------------------------------------------------------- 1 | name: Get Deployment Bundle Info 2 | description: Get deployment bundle info from Astronomer API 3 | inputs: 4 | deployment_id: 5 | description: The deployment ID 6 | required: true 7 | organization_id: 8 | description: The organization ID 9 | required: true 10 | astro_api_token: 11 | description: The Astronomer API token 12 | required: true 13 | astronomer_host: 14 | description: The Astronomer host 15 | required: true 16 | expected_status_code: 17 | description: The expected status code 18 | default: 200 19 | 20 | outputs: 21 | desired_bundle_version: 22 | description: The desired bundle version 23 | value: ${{ steps.get-deployment-info.outputs.desired_bundle_version }} 24 | bundle_type: 25 | description: The bundle type 26 | value: ${{ steps.get-deployment-info.outputs.bundle_type }} 27 | updated_at: 28 | description: The timestamp at which the bundle was last updated 29 | value: ${{ steps.get-deployment-info.outputs.updated_at }} 30 | 31 | runs: 32 | using: "composite" 33 | steps: 34 | - name: Get Deployment Info 35 | id: get-deployment-info 36 | shell: bash 37 | run: | 38 | STATUS_CODE=$(curl -s -w "%{http_code}" -o response.json -H "Authorization: Bearer ${{ inputs.astro_api_token }}" "https://api.${{ inputs.astronomer_host }}/v1alpha1/organizations/${{ inputs.organization_id }}/deployments/${{ inputs.deployment_id }}/deploys") 39 | if [[ $STATUS_CODE -ne ${{ inputs.expected_status_code }} ]]; then 40 | echo "Failed to get expected status code from GET Deployment API. Status code: $STATUS_CODE" 41 | exit 1 42 | fi 43 | if [[ $(cat response.json | jq -r '.deploys | length') -eq 0 ]]; then 44 | echo "No deploys found for the deployment: ${{ inputs.deployment_id }}" 45 | exit 1 46 | fi 47 | 48 | # sort by updatedAt to fetch the latest deploy object 49 | cat response.json | jq '.deploys | sort_by(.updatedAt)' > response_sorted.json 50 | 51 | desired_bundle_version=$(cat response_sorted.json | jq -r '.[] | select(.type == "BUNDLE" and .bundles[0].bundleType == "dbt") | .bundles[0].desiredVersion' | head -n 1) 52 | bundle_type=$(cat response_sorted.json | jq -r '.[] | select(.type == "BUNDLE" and .bundles[0].bundleType == "dbt") | .bundles[0].bundleType' | head -n 1) 53 | updated_at=$(cat response_sorted.json | jq -r '.[] | select(.type == "BUNDLE" and .bundles[0].bundleType == "dbt") | .updatedAt' | head -n 1) 54 | 55 | echo "desired_bundle_version=$desired_bundle_version" >> $GITHUB_OUTPUT 56 | echo "bundle_type=$bundle_type" >> $GITHUB_OUTPUT 57 | echo "updated_at=$updated_at" >> $GITHUB_OUTPUT 58 | -------------------------------------------------------------------------------- /.github/workflows/e2e/get_deployment_info/action.yaml: -------------------------------------------------------------------------------- 1 | name: Get Deployment Info 2 | description: Get deployment info from Astronomer API 3 | inputs: 4 | deployment_id: 5 | description: The deployment ID 6 | required: true 7 | organization_id: 8 | description: The organization ID 9 | required: true 10 | astro_api_token: 11 | description: The Astronomer API token 12 | required: true 13 | astronomer_host: 14 | description: The Astronomer host 15 | required: true 16 | expected_status_code: 17 | description: The expected status code 18 | default: 200 19 | 20 | outputs: 21 | desired_dag_tarball_version: 22 | description: The desired DAG tarball version 23 | value: ${{ steps.get-deployment-info.outputs.desired_dag_tarball_version }} 24 | desired_image_version: 25 | description: The image version 26 | value: ${{ steps.get-deployment-info.outputs.desired_image_version }} 27 | hibernation_spec: 28 | description: The hibernation spec 29 | value: ${{ steps.get-deployment-info.outputs.hibernation_spec }} 30 | 31 | runs: 32 | using: "composite" 33 | steps: 34 | - name: Get Deployment Info 35 | id: get-deployment-info 36 | shell: bash 37 | run: | 38 | STATUS_CODE=$(curl -s -w "%{http_code}" -o response.json -H "Authorization: Bearer ${{ inputs.astro_api_token }}" "https://api.${{ inputs.astronomer_host }}/v1alpha1/organizations/${{ inputs.organization_id }}/deployments/${{ inputs.deployment_id }}") 39 | if [[ $STATUS_CODE -ne ${{ inputs.expected_status_code }} ]]; then 40 | echo "Failed to get expected status code from GET Deployment API. Status code: $STATUS_CODE" 41 | exit 1 42 | fi 43 | desired_dag_tarball_version=$(cat response.json | jq -r '.desiredDagTarballVersion') 44 | desired_image_version=$(cat response.json | jq -r '.desiredImageVersion') 45 | hibernation_spec=$(cat response.json | jq -rc '.scalingSpec.hibernationSpec') 46 | echo "desired_dag_tarball_version=$desired_dag_tarball_version" >> $GITHUB_OUTPUT 47 | echo "desired_image_version=$desired_image_version" >> $GITHUB_OUTPUT 48 | echo "hibernation_spec=$hibernation_spec" >> $GITHUB_OUTPUT 49 | -------------------------------------------------------------------------------- /.github/workflows/e2e/validate_deployment/action.yaml: -------------------------------------------------------------------------------- 1 | name: Validate Deployment Deploy Versions 2 | description: Validate deployment info from Astronomer API 3 | 4 | inputs: 5 | is_dag_only_deploy: 6 | description: Whether the deploy operation was DAG-only 7 | default: false 8 | is_no_deploy: 9 | description: If the deploy test was a no-op 10 | default: false 11 | is_wake_on_deploy: 12 | description: If the deploy test was a wake-on-deploy 13 | default: false 14 | dag_tarball_version_before: 15 | description: The desired DAG tarball version before the test 16 | required: true 17 | image_version_before: 18 | description: The image version before the test 19 | required: true 20 | dag_tarball_version_after: 21 | description: The desired DAG tarball version after the test 22 | required: true 23 | image_version_after: 24 | description: The image version after the test 25 | required: true 26 | hibernation_spec_before: 27 | description: The hibernation spec before the test 28 | required: true 29 | hibernation_spec_after: 30 | description: The hibernation spec after the test 31 | required: true 32 | 33 | runs: 34 | using: "composite" 35 | steps: 36 | - name: Validate Deployment 37 | id: validate-deployment 38 | shell: bash 39 | run: | 40 | 41 | if [[ "${{ inputs.is_no_deploy }}" == "true" ]]; then 42 | if [[ "${{ inputs.dag_tarball_version_before }}" == "${{ inputs.dag_tarball_version_after }}" && "${{ inputs.image_version_before }}" == "${{ inputs.image_version_after }}" ]]; then 43 | echo "Deploy Action validation succeeded: no deploy operation" 44 | exit 0 45 | fi 46 | echo "Deploy Action validation failed: deploy operation was not a no-op" 47 | exit 1 48 | fi 49 | 50 | # If it's dag only deploy then validate that only desiredDagTarballVersion was updated 51 | if [[ "${{ inputs.is_dag_only_deploy }}" == "true" ]]; then 52 | if [[ "${{ inputs.dag_tarball_version_before }}" != "${{ inputs.dag_tarball_version_after }}" && "${{ inputs.image_version_before }}" == "${{ inputs.image_version_after }}" ]]; then 53 | echo "Deploy Action validation succeeded: desiredDagTarballVersion only updated" 54 | exit 0 55 | fi 56 | echo "Deploy Action validation failed" 57 | exit 1 58 | fi 59 | 60 | # If it's not dag only deploy then validate that both desiredDagTarballVersion and imageVersion were updated 61 | if [[ "${{ inputs.dag_tarball_version_before }}" != "${{ inputs.dag_tarball_version_after }}" && "${{ inputs.image_version_before }}" != "${{ inputs.image_version_after }}" ]]; then 62 | echo "Deploy Action validation succeeded: desiredDagTarballVersion and imageVersion updated" 63 | exit 0 64 | fi 65 | echo "Deploy Action validation failed: neither desiredDagTarballVersion or imageVersion were updated after the test" 66 | exit 1 67 | 68 | - name: Validate Hibernation Spec 69 | id: validate-hibernation-spec 70 | shell: bash 71 | run: | 72 | 73 | if [[ "${{ inputs.is_wake_on_deploy }}" == "true" ]]; then 74 | if [[ '${{ inputs.hibernation_spec_before }}' == '${{ inputs.hibernation_spec_after }}' ]]; then 75 | echo "Hibernation Spec validation succeeded: hibernation spec was not updated after the test" 76 | exit 0 77 | fi 78 | echo "Hibernation Spec validation failed: hibernation spec updated" 79 | exit 1 80 | fi 81 | -------------------------------------------------------------------------------- /.github/workflows/lint_yaml.yaml: -------------------------------------------------------------------------------- 1 | name: Lint YAML 2 | on: 3 | pull_request: 4 | branches: ['main'] 5 | jobs: 6 | lint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | - name: yaml-lint 11 | uses: ibiqlik/action-yamllint@v3 12 | with: 13 | file_or_dir: action.yaml 14 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: E2E Tests for Astro Deploy Action 2 | 3 | on: 4 | push: 5 | 6 | env: 7 | ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} 8 | 9 | jobs: 10 | redact-inputs: 11 | name: Redact Inputs 12 | runs-on: ubuntu-latest 13 | environment: e2e-test 14 | steps: 15 | - name: Checkout code 16 | uses: actions/checkout@v4 17 | 18 | - name: Redact Inputs 19 | run: | 20 | 21 | # add-mask will redact the value of the input, but will still log the value, 22 | # hence using a known workaround: https://github.com/actions/runner/issues/643 23 | WORKSPACE_ID=$(jq -r '.inputs.workspace_id' $GITHUB_EVENT_PATH) 24 | echo ::add-mask::$WORKSPACE_ID 25 | ORG_ID=$(jq -r '.inputs.org_id' $GITHUB_EVENT_PATH) 26 | echo ::add-mask::$ORG_ID 27 | ASTRONOMER_HOST=$(jq -r '.inputs.astronomer_host' $GITHUB_EVENT_PATH) 28 | echo ::add-mask::$ASTRONOMER_HOST 29 | TOKEN=$(jq -r '.inputs.token' $GITHUB_EVENT_PATH) 30 | echo ::add-mask::$TOKEN 31 | 32 | # Create test deployments would use the template files and generate deployments with unique names 33 | create-test-deployments: 34 | name: Create Deployment 35 | needs: redact-inputs 36 | runs-on: ubuntu-latest 37 | strategy: 38 | matrix: 39 | deployment: [deployment-hibernate.yaml, deployment.yaml] 40 | outputs: 41 | DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.DEPLOYMENT_ID }} 42 | HIBERNATE_DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.HIBERNATE_DEPLOYMENT_ID }} 43 | steps: 44 | - name: Checkout code 45 | uses: actions/checkout@v4 46 | 47 | - name: Get Astro Environment Info 48 | id: get-astro-env-info 49 | uses: ./.github/workflows/e2e/get_astro_env_info 50 | with: 51 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 52 | input_organization_id: ${{ github.event.inputs.org_id }} 53 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 54 | input_astro_api_token: ${{ github.event.inputs.token }} 55 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 56 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 57 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 58 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 59 | 60 | - name: Install dependencies 61 | run: | 62 | 63 | wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq 64 | 65 | curl -sSL https://install.astronomer.io | sudo bash -s 66 | astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 67 | astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }} 68 | 69 | # Generate a random test_uid and set it as an environment variable, so we can append it to the deployment name 70 | test_uid=$(uuidgen) 71 | echo "test_uid=$test_uid" >> $GITHUB_ENV 72 | 73 | - name: Create Deployment 74 | id: create-deployment 75 | run: | 76 | 77 | # Update the deployment name to maintain uniqueness 78 | sed -i "s| name:.*| name: deploy-action-e2e-${{ env.test_uid }}|g" e2e-setup/deployment-templates/${{ matrix.deployment }} 79 | 80 | DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -) 81 | if [[ "${{ matrix.deployment }}" == "deployment.yaml" ]]; then 82 | echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT 83 | else 84 | echo "HIBERNATE_DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT 85 | fi 86 | default-deploy-tests: 87 | name: Default Deploy Test 88 | runs-on: ubuntu-latest 89 | needs: [create-test-deployments] 90 | strategy: 91 | max-parallel: 1 92 | matrix: 93 | deployment_id: 94 | [ 95 | "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", 96 | "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", 97 | ] 98 | deploy_type: [dags, image-and-dags] 99 | steps: 100 | - name: Checkout code 101 | uses: actions/checkout@v4 102 | 103 | - name: Get Astro Environment Info 104 | id: get-astro-env-info 105 | uses: ./.github/workflows/e2e/get_astro_env_info 106 | with: 107 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 108 | input_organization_id: ${{ github.event.inputs.org_id }} 109 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 110 | input_astro_api_token: ${{ github.event.inputs.token }} 111 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 112 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 113 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 114 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 115 | 116 | - name: Install dependencies 117 | run: | 118 | 119 | sudo apt-get install jq 120 | 121 | # we need to pre-install the CLI to set the context 122 | curl -sSL https://install.astronomer.io | sudo bash -s 123 | 124 | - name: Mock git commands 125 | run: | 126 | 127 | # if deploy_type is dags, then use dags-deploy-git.sh as mock otherwise use image-deploy-git.sh as mock 128 | if [[ "${{ matrix.deploy_type }}" == "dags" ]]; then 129 | mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git 130 | else 131 | mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git 132 | fi 133 | chmod +x /usr/local/bin/git 134 | cp /usr/local/bin/astro /usr/local/bin/astro-original 135 | mv e2e-setup/mocks/astro-deploy.sh /usr/local/bin/astro 136 | chmod +x /usr/local/bin/astro 137 | 138 | - name: Set CLI context 139 | run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 140 | 141 | - name: Get Deployment Info Before Test 142 | id: get-deployment-before 143 | uses: ./.github/workflows/e2e/get_deployment_info 144 | with: 145 | deployment_id: ${{ matrix.deployment_id }} 146 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 147 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 148 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 149 | 150 | - name: Run Deploy Action 151 | uses: ./ 152 | with: 153 | deployment-id: ${{ matrix.deployment_id }} 154 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 155 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 156 | 157 | - name: Get Deployment Info After Test 158 | id: get-deployment-after 159 | uses: ./.github/workflows/e2e/get_deployment_info 160 | with: 161 | deployment_id: ${{ matrix.deployment_id }} 162 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 163 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 164 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 165 | 166 | - name: Validate Deploy Action 167 | uses: ./.github/workflows/e2e/validate_deployment 168 | with: 169 | is_wake_on_deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 170 | is_dag_only_deploy: ${{ matrix.deploy_type == 'dags' }} 171 | dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} 172 | image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} 173 | hibernation_spec_before: ${{ steps.get-deployment-before.outputs.hibernation_spec }} 174 | dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} 175 | image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} 176 | hibernation_spec_after: ${{ steps.get-deployment-after.outputs.hibernation_spec }} 177 | 178 | # DAG Deploy test would test the DAG only deploy functionality in deploy action 179 | dag-deploy-test: 180 | name: DAG Deploy Test 181 | runs-on: ubuntu-latest 182 | needs: [default-deploy-tests, create-test-deployments] 183 | strategy: 184 | matrix: 185 | deployment_id: 186 | [ 187 | "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", 188 | "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", 189 | ] 190 | steps: 191 | - name: Checkout code 192 | uses: actions/checkout@v4 193 | 194 | - name: Get Astro Environment Info 195 | id: get-astro-env-info 196 | uses: ./.github/workflows/e2e/get_astro_env_info 197 | with: 198 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 199 | input_organization_id: ${{ github.event.inputs.org_id }} 200 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 201 | input_astro_api_token: ${{ github.event.inputs.token }} 202 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 203 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 204 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 205 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 206 | 207 | - name: Mock git commands 208 | run: | 209 | mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git 210 | chmod +x /usr/local/bin/git 211 | 212 | - name: Install dependencies 213 | run: | 214 | 215 | sudo apt-get install jq 216 | # we need to pre-install the CLI to set the context 217 | curl -sSL https://install.astronomer.io | sudo bash -s 218 | 219 | - name: Set CLI context 220 | run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 221 | 222 | - name: Get Deployment Info Before Test 223 | id: get-deployment-before 224 | uses: ./.github/workflows/e2e/get_deployment_info 225 | with: 226 | deployment_id: ${{ matrix.deployment_id }} 227 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 228 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 229 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 230 | 231 | - name: Deploy to Astro 232 | uses: ./ 233 | with: 234 | deployment-id: ${{ matrix.deployment_id }} 235 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 236 | parse: true 237 | root-folder: e2e-setup/astro-project 238 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 239 | 240 | - name: Get Deployment Info After Test 241 | id: get-deployment-after 242 | uses: ./.github/workflows/e2e/get_deployment_info 243 | with: 244 | deployment_id: ${{ matrix.deployment_id }} 245 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 246 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 247 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 248 | 249 | - name: Validate Deploy Action 250 | uses: ./.github/workflows/e2e/validate_deployment 251 | with: 252 | is_wake_on_deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 253 | is_dag_only_deploy: true 254 | dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} 255 | image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} 256 | hibernation_spec_before: ${{ steps.get-deployment-before.outputs.hibernation_spec }} 257 | dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} 258 | image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} 259 | hibernation_spec_after: ${{ steps.get-deployment-after.outputs.hibernation_spec }} 260 | 261 | pytests-test: 262 | name: Pytest Test 263 | runs-on: ubuntu-latest 264 | needs: [dag-deploy-test, create-test-deployments] 265 | strategy: 266 | matrix: 267 | deployment_id: 268 | [ 269 | "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", 270 | "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", 271 | ] 272 | steps: 273 | - name: Checkout code 274 | uses: actions/checkout@v4 275 | 276 | - name: Get Astro Environment Info 277 | id: get-astro-env-info 278 | uses: ./.github/workflows/e2e/get_astro_env_info 279 | with: 280 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 281 | input_organization_id: ${{ github.event.inputs.org_id }} 282 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 283 | input_astro_api_token: ${{ github.event.inputs.token }} 284 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 285 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 286 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 287 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 288 | 289 | - name: Install dependencies 290 | run: | 291 | 292 | sudo apt-get install jq 293 | 294 | # we need to pre-install the CLI to set the context 295 | curl -sSL https://install.astronomer.io | sudo bash -s 296 | 297 | - name: Set CLI context 298 | run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 299 | 300 | - name: Mock git commands for DAG Deploy 301 | run: | 302 | 303 | # set to dag deploy git mock, with deploy-type set to image-and-dags to test that it will do a image deploy 304 | mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git 305 | chmod +x /usr/local/bin/git 306 | 307 | - name: Get Deployment Info Before Test 308 | id: get-deployment-before 309 | uses: ./.github/workflows/e2e/get_deployment_info 310 | with: 311 | deployment_id: ${{ matrix.deployment_id }} 312 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 313 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 314 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 315 | 316 | - name: Run Pytest 317 | uses: ./ 318 | with: 319 | deployment-id: ${{ matrix.deployment_id }} 320 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 321 | pytest: true 322 | action: deploy 323 | deploy-type: image-and-dags 324 | root-folder: e2e-setup/astro-project 325 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 326 | 327 | - name: Get Deployment Info After Test 328 | id: get-deployment-after 329 | uses: ./.github/workflows/e2e/get_deployment_info 330 | with: 331 | deployment_id: ${{ matrix.deployment_id }} 332 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 333 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 334 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 335 | 336 | - name: Validate Deploy Action 337 | uses: ./.github/workflows/e2e/validate_deployment 338 | with: 339 | is_wake_on_deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 340 | dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} 341 | image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} 342 | hibernation_spec_before: ${{ steps.get-deployment-before.outputs.hibernation_spec }} 343 | dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} 344 | image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} 345 | hibernation_spec_after: ${{ steps.get-deployment-after.outputs.hibernation_spec }} 346 | 347 | infer-deploy: 348 | name: Infer Deploy Test 349 | runs-on: ubuntu-latest 350 | needs: [pytests-test, create-test-deployments] 351 | strategy: 352 | max-parallel: 1 353 | matrix: 354 | deployment_id: 355 | [ 356 | "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", 357 | "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", 358 | ] 359 | deploy_type: [dags, image-and-dags] 360 | steps: 361 | - name: Checkout code 362 | uses: actions/checkout@v4 363 | 364 | - name: Get Astro Environment Info 365 | id: get-astro-env-info 366 | uses: ./.github/workflows/e2e/get_astro_env_info 367 | with: 368 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 369 | input_organization_id: ${{ github.event.inputs.org_id }} 370 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 371 | input_astro_api_token: ${{ github.event.inputs.token }} 372 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 373 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 374 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 375 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 376 | 377 | - name: Mock git commands 378 | run: | 379 | 380 | # if deploy_type is dags, then use dags-deploy-git.sh as mock otherwise use image-deploy-git.sh as mock 381 | if [[ "${{ matrix.deploy_type }}" == "dags" ]]; then 382 | mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git 383 | else 384 | mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git 385 | fi 386 | chmod +x /usr/local/bin/git 387 | 388 | - name: Install dependencies 389 | run: | 390 | 391 | sudo apt-get install jq 392 | 393 | # we need to pre-install the CLI to set the context 394 | curl -sSL https://install.astronomer.io | sudo bash -s 395 | 396 | - name: Set CLI context 397 | run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 398 | 399 | - name: Get Deployment Info Before Test 400 | id: get-deployment-before 401 | uses: ./.github/workflows/e2e/get_deployment_info 402 | with: 403 | deployment_id: ${{ matrix.deployment_id }} 404 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 405 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 406 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 407 | 408 | - name: Run Deploy Action 409 | uses: ./ 410 | with: 411 | deployment-id: ${{ matrix.deployment_id }} 412 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 413 | parse: true 414 | action: deploy 415 | root-folder: e2e-setup/astro-project 416 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 417 | 418 | - name: Get Deployment Info After Test 419 | id: get-deployment-after 420 | uses: ./.github/workflows/e2e/get_deployment_info 421 | with: 422 | deployment_id: ${{ matrix.deployment_id }} 423 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 424 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 425 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 426 | 427 | - name: Validate Deploy Action 428 | uses: ./.github/workflows/e2e/validate_deployment 429 | with: 430 | is_wake_on_deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 431 | is_dag_only_deploy: ${{ matrix.deploy_type == 'dags' }} 432 | dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} 433 | image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} 434 | hibernation_spec_before: ${{ steps.get-deployment-before.outputs.hibernation_spec }} 435 | dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} 436 | image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} 437 | hibernation_spec_after: ${{ steps.get-deployment-after.outputs.hibernation_spec }} 438 | 439 | no-deploy-test: 440 | name: No Deploy Test 441 | runs-on: ubuntu-latest 442 | needs: [infer-deploy, create-test-deployments] 443 | strategy: 444 | matrix: 445 | deployment_id: 446 | [ 447 | "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", 448 | "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", 449 | ] 450 | deploy_type: [infer, dbt, dags, image-and-dags] 451 | steps: 452 | - name: Checkout code 453 | uses: actions/checkout@v4 454 | 455 | - name: Get Astro Environment Info 456 | id: get-astro-env-info 457 | uses: ./.github/workflows/e2e/get_astro_env_info 458 | with: 459 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 460 | input_organization_id: ${{ github.event.inputs.org_id }} 461 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 462 | input_astro_api_token: ${{ github.event.inputs.token }} 463 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 464 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 465 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 466 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 467 | 468 | - name: Setup dependencies 469 | run: | 470 | 471 | sudo apt-get install jq 472 | 473 | # we need to pre-install the CLI to set the context 474 | curl -sSL https://install.astronomer.io | sudo bash -s 475 | 476 | - name: Set CLI context 477 | run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 478 | 479 | - name: Mock git commands for DAG Deploy 480 | run: | 481 | 482 | # set to no deploy git mock, to simulate no interested file change scenario 483 | mv e2e-setup/mocks/no-deploy-git.sh /usr/local/bin/git 484 | chmod +x /usr/local/bin/git 485 | 486 | - name: Get Deployment Info Before Test 487 | id: get-deployment-before 488 | if: ${{ matrix.deploy_type != 'dbt' }} 489 | uses: ./.github/workflows/e2e/get_deployment_info 490 | with: 491 | deployment_id: ${{ matrix.deployment_id }} 492 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 493 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 494 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 495 | 496 | - name: Run Deploy Action 497 | uses: ./ 498 | with: 499 | deployment-id: ${{ matrix.deployment_id }} 500 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 501 | action: deploy 502 | deploy-type: ${{ matrix.deploy_type }} 503 | root-folder: ${{ matrix.deploy_type == 'dbt' && 'e2e-setup/dbt' || 'e2e-setup/astro-project' }} 504 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 505 | 506 | - name: Get Deployment Info After Test 507 | id: get-deployment-after 508 | if: ${{ matrix.deploy_type != 'dbt' }} 509 | uses: ./.github/workflows/e2e/get_deployment_info 510 | with: 511 | deployment_id: ${{ matrix.deployment_id }} 512 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 513 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 514 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 515 | 516 | - name: Validate no image or dag deploy happened 517 | if: ${{ matrix.deploy_type != 'dbt' }} 518 | uses: ./.github/workflows/e2e/validate_deployment 519 | with: 520 | is_wake_on_deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 521 | is_no_deploy: true 522 | dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} 523 | image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} 524 | hibernation_spec_before: ${{ steps.get-deployment-before.outputs.hibernation_spec }} 525 | dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} 526 | image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} 527 | hibernation_spec_after: ${{ steps.get-deployment-after.outputs.hibernation_spec }} 528 | 529 | - name: Get DBT bundle info 530 | id: get-dbt-bundle-info 531 | if: ${{ matrix.deploy_type == 'dbt' }} 532 | uses: ./.github/workflows/e2e/get_bundle_info 533 | with: 534 | deployment_id: ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} 535 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 536 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 537 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 538 | 539 | - name: Validate no DBT deploy happened 540 | id: validate-dbt-deploy-action 541 | if: ${{ matrix.deploy_type == 'dbt' }} 542 | shell: bash 543 | run: | 544 | 545 | if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" != "" ]]; then 546 | echo "DBT Deploy Action validation failed: DBT bundle found, but there should be no deploy" 547 | exit 1 548 | fi 549 | 550 | echo "DBT Deploy Action validation succeeded: DBT bundle not found" 551 | exit 0 552 | 553 | custom-docker-image-test: 554 | name: Custom Docker Image Test 555 | runs-on: ubuntu-latest 556 | needs: [no-deploy-test, create-test-deployments] 557 | strategy: 558 | matrix: 559 | deployment_id: 560 | [ 561 | "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", 562 | "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", 563 | ] 564 | steps: 565 | - name: Checkout code 566 | uses: actions/checkout@v4 567 | 568 | - name: Get Astro Environment Info 569 | id: get-astro-env-info 570 | uses: ./.github/workflows/e2e/get_astro_env_info 571 | with: 572 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 573 | input_organization_id: ${{ github.event.inputs.org_id }} 574 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 575 | input_astro_api_token: ${{ github.event.inputs.token }} 576 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 577 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 578 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 579 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 580 | 581 | - name: Install dependencies 582 | run: | 583 | 584 | sudo apt-get install jq 585 | 586 | # we need to pre-install the CLI to set the context 587 | curl -sSL https://install.astronomer.io | sudo bash -s 588 | 589 | - name: Build Docker image 590 | run: cd e2e-setup/astro-project && docker build -t custom-image:latest . 591 | 592 | - name: Set CLI context 593 | run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 594 | 595 | - name: Get Deployment Info Before Test 596 | id: get-deployment-before 597 | uses: ./.github/workflows/e2e/get_deployment_info 598 | with: 599 | deployment_id: ${{ matrix.deployment_id }} 600 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 601 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 602 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 603 | 604 | - name: Deploy to Astro 605 | uses: ./ 606 | with: 607 | deployment-id: ${{ matrix.deployment_id }} 608 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 609 | image-name: custom-image:latest 610 | action: deploy 611 | deploy-type: image-and-dags 612 | root-folder: e2e-setup/astro-project 613 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 614 | 615 | - name: Get Deployment Info After Test 616 | id: get-deployment-after 617 | uses: ./.github/workflows/e2e/get_deployment_info 618 | with: 619 | deployment_id: ${{ matrix.deployment_id }} 620 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 621 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 622 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 623 | 624 | - name: Validate Deploy Action 625 | uses: ./.github/workflows/e2e/validate_deployment 626 | with: 627 | is_wake_on_deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 628 | dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} 629 | image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} 630 | hibernation_spec_before: ${{ steps.get-deployment-before.outputs.hibernation_spec }} 631 | dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desiredDagTarballVersion }} 632 | image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} 633 | hibernation_spec_after: ${{ steps.get-deployment-after.outputs.hibernation_spec }} 634 | 635 | # DBT Deploy test would test the DAG only deploy functionality in deploy action 636 | dbt-deploy-test: 637 | name: DBT Deploy Test 638 | runs-on: ubuntu-latest 639 | needs: [custom-docker-image-test, create-test-deployments] 640 | strategy: 641 | matrix: 642 | deployment_id: 643 | [ 644 | "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", 645 | "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", 646 | ] 647 | steps: 648 | - name: Checkout code 649 | uses: actions/checkout@v4 650 | 651 | - name: Get Astro Environment Info 652 | id: get-astro-env-info 653 | uses: ./.github/workflows/e2e/get_astro_env_info 654 | with: 655 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 656 | input_organization_id: ${{ github.event.inputs.org_id }} 657 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 658 | input_astro_api_token: ${{ github.event.inputs.token }} 659 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 660 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 661 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 662 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 663 | 664 | - name: Mock git commands 665 | run: | 666 | mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git 667 | chmod +x /usr/local/bin/git 668 | 669 | - name: Install dependencies 670 | id: setup 671 | run: | 672 | 673 | sudo apt-get install jq 674 | # we need to pre-install the CLI to set the context 675 | curl -sSL https://install.astronomer.io | sudo bash -s 676 | 677 | test_start_time=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ") 678 | echo "test_start_time=$test_start_time" >> $GITHUB_OUTPUT 679 | 680 | - name: Set CLI context 681 | run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 682 | 683 | - name: DBT Deploy to Astro 684 | uses: ./ 685 | with: 686 | deployment-id: ${{ matrix.deployment_id }} 687 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 688 | parse: true 689 | root-folder: e2e-setup/dbt 690 | deploy-type: dbt 691 | description: "test-dbt-deploy-action" 692 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 693 | 694 | - name: Get DBT bundle info 695 | id: get-dbt-bundle-info 696 | uses: ./.github/workflows/e2e/get_bundle_info 697 | with: 698 | deployment_id: ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} 699 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 700 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 701 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 702 | 703 | - name: Validate DBT Deploy Action 704 | id: validate-dbt-deploy-action 705 | shell: bash 706 | run: | 707 | if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then 708 | echo "DBT Deploy Action validation failed: DBT bundle not found" 709 | exit 1 710 | fi 711 | 712 | updated_at=$(date -u -d "${{ steps.get-dbt-bundle-info.outputs.updated_at }}" +"%s.%N") 713 | test_start_time=$(date -u -d "${{ steps.setup.outputs.test_start_time }}" +"%s.%N") 714 | if [[ $(echo "$updated_at > $test_start_time" | bc -l) ]]; then 715 | echo "DBT Deploy Action validation succeeded: DBT bundle found" 716 | exit 0 717 | fi 718 | 719 | echo "DBT Deploy Action validation failed: DBT bundle was not updated" 720 | exit 1 721 | 722 | # Deployment preview tests can run in parallel to above tests since it operates on a different deployment, so it won't interfere with the above tests 723 | deployment-preview-test: 724 | name: Deployment Preview Test 725 | runs-on: ubuntu-latest 726 | needs: [create-test-deployments] 727 | strategy: 728 | matrix: 729 | deployment_id: 730 | [ 731 | "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", 732 | "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", 733 | ] 734 | steps: 735 | - name: Checkout code 736 | uses: actions/checkout@v4 737 | 738 | - name: Get Astro Environment Info 739 | id: get-astro-env-info 740 | uses: ./.github/workflows/e2e/get_astro_env_info 741 | with: 742 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 743 | input_organization_id: ${{ github.event.inputs.org_id }} 744 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 745 | input_astro_api_token: ${{ github.event.inputs.token }} 746 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 747 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 748 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 749 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 750 | 751 | - name: Install dependencies 752 | id: setup 753 | run: | 754 | 755 | sudo apt-get install jq 756 | 757 | # we need to pre-install the CLI to set the context 758 | curl -sSL https://install.astronomer.io | sudo bash -s 759 | 760 | test_start_time=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ") 761 | echo "test_start_time=$test_start_time" >> $GITHUB_OUTPUT 762 | 763 | - name: Set CLI context 764 | run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 765 | 766 | - name: Create Deployment Preview 767 | id: create-deployment-preview 768 | uses: ./ 769 | with: 770 | deployment-id: ${{ matrix.deployment_id }} 771 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 772 | action: create-deployment-preview 773 | preview-name: test-preview-${{ matrix.deployment_id }} 774 | root-folder: e2e-setup/astro-project 775 | # TODO: allow copying airflow resources, once we add wait in the deploy action for deployment preview to be healthy 776 | copy-connections: false 777 | copy-airflow-variables: false 778 | copy-pools: false 779 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 780 | 781 | - name: Get Deployment Info After Test 782 | id: get-deployment-after-create-preview 783 | uses: ./.github/workflows/e2e/get_deployment_info 784 | with: 785 | deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} 786 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 787 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 788 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 789 | 790 | - name: Validate Deploy Action 791 | id: validate-deployment-preview-create 792 | uses: ./.github/workflows/e2e/validate_deployment 793 | with: 794 | is_wake_on_deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 795 | dag_tarball_version_before: "" 796 | image_version_before: "12.7.1" 797 | hibernation_spec_before: '{"schedules":[{"isEnabled":true,"hibernateAtCron":"0 * * * *","wakeAtCron":"1 * * * *"},{"isEnabled":true,"hibernateAtCron":"1 * * * *","wakeAtCron":"0 * * * *"}]}' 798 | dag_tarball_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_dag_tarball_version }} 799 | image_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_image_version }} 800 | hibernation_spec_after: ${{ steps.get-deployment-after-create-preview.outputs.hibernation_spec }} 801 | 802 | - name: Mock git commands 803 | run: | 804 | mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git 805 | chmod +x /usr/local/bin/git 806 | 807 | - name: Get Deployment Info Before Test 808 | id: get-deployment-before-deploy 809 | uses: ./.github/workflows/e2e/get_deployment_info 810 | with: 811 | deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} 812 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 813 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 814 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 815 | 816 | - name: Deploy to Deployment Preview 817 | id: deployment-preview-deploy 818 | uses: ./ 819 | with: 820 | deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} 821 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 822 | action: deploy-deployment-preview 823 | deploy-type: image-and-dags 824 | root-folder: e2e-setup/astro-project 825 | preview-name: test-preview-${{ matrix.deployment_id }} 826 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 827 | 828 | - name: Get Deployment Info After Test 829 | id: get-deployment-after-deploy-preview 830 | uses: ./.github/workflows/e2e/get_deployment_info 831 | with: 832 | deployment_id: ${{ steps.deployment-preview-deploy.outputs.preview-id }} 833 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 834 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 835 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 836 | 837 | - name: Validate Deploy Action 838 | id: validate-deployment-preview-deploy 839 | uses: ./.github/workflows/e2e/validate_deployment 840 | with: 841 | is_wake_on_deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 842 | dag_tarball_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_dag_tarball_version }} 843 | image_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_image_version }} 844 | hibernation_spec_before: ${{ steps.get-deployment-before-deploy.outputs.hibernation_spec }} 845 | dag_tarball_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_dag_tarball_version }} 846 | image_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_image_version }} 847 | hibernation_spec_after: ${{ steps.get-deployment-after-deploy-preview.outputs.hibernation_spec }} 848 | 849 | - name: Mock git commands for DAG Deploy 850 | run: | 851 | mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git 852 | chmod +x /usr/local/bin/git 853 | 854 | - name: Get Deployment Info Before DAG Deploy Test 855 | id: get-deployment-before-dag-deploy 856 | uses: ./.github/workflows/e2e/get_deployment_info 857 | with: 858 | deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} 859 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 860 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 861 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 862 | 863 | - name: DAG Deploy to Astro 864 | id: deployment-preview-dag-deploy 865 | uses: ./ 866 | with: 867 | deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} 868 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 869 | action: deploy-deployment-preview 870 | parse: true 871 | deploy-type: dags-only 872 | root-folder: e2e-setup/astro-project 873 | preview-name: test-preview-${{ matrix.deployment_id }} 874 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 875 | 876 | - name: Get Deployment Info After DAG Deploy Test 877 | id: get-deployment-after-dag-deploy 878 | uses: ./.github/workflows/e2e/get_deployment_info 879 | with: 880 | deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} 881 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 882 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 883 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 884 | 885 | - name: Validate DAG Deploy Action 886 | id: validate-dag-deploy-action 887 | uses: ./.github/workflows/e2e/validate_deployment 888 | with: 889 | is_wake_on_deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 890 | is_dag_only_deploy: true 891 | dag_tarball_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_dag_tarball_version }} 892 | image_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_image_version }} 893 | hibernation_spec_before: ${{ steps.get-deployment-before-dag-deploy.outputs.hibernation_spec }} 894 | dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }} 895 | image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }} 896 | hibernation_spec_after: ${{ steps.get-deployment-after-dag-deploy.outputs.hibernation_spec }} 897 | 898 | - name: Mock git commands for DBT Deploy 899 | run: | 900 | mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git 901 | chmod +x /usr/local/bin/git 902 | 903 | - name: DBT Deploy to Astro 904 | id: deployment-preview-dbt-deploy 905 | uses: ./ 906 | with: 907 | deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} 908 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 909 | action: deploy-deployment-preview 910 | deploy-type: dbt 911 | description: "test-dbt-deploy-action" 912 | root-folder: e2e-setup/dbt 913 | preview-name: test-preview-${{ matrix.deployment_id }} 914 | wake-on-deploy: ${{ matrix.deployment_id == needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 915 | 916 | - name: Get DBT bundle info 917 | id: get-dbt-bundle-info 918 | uses: ./.github/workflows/e2e/get_bundle_info 919 | with: 920 | deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} 921 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 922 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 923 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 924 | 925 | - name: Validate DBT Deploy Action 926 | id: validate-dbt-deploy-action 927 | shell: bash 928 | run: | 929 | if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then 930 | echo "DBT Deploy Action validation failed: DBT bundle not found" 931 | exit 1 932 | fi 933 | 934 | updated_at=$(date -u -d "${{ steps.get-dbt-bundle-info.outputs.updated_at }}" +"%s.%N") 935 | test_start_time=$(date -u -d "${{ steps.setup.outputs.test_start_time }}" +"%s.%N") 936 | if [[ $(echo "$updated_at > $test_start_time" | bc -l) ]]; then 937 | echo "DBT Deploy Action validation succeeded: DBT bundle found" 938 | exit 0 939 | fi 940 | 941 | echo "DBT Deploy Action validation failed: DBT bundle was not updated" 942 | exit 1 943 | 944 | - name: Delete Deployment Preview 945 | id: delete-deployment-preview 946 | # ensure that we always try delete the deployment preview even if the previous steps fail 947 | if: always() 948 | uses: ./ 949 | with: 950 | deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} 951 | workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} 952 | action: delete-deployment-preview 953 | preview-name: test-preview-${{ matrix.deployment_id }} 954 | 955 | - name: Validate Deployment Preview Deletion 956 | id: validate-deployment-preview-deletion 957 | # ensure that we always try delete the deployment preview even if the previous steps fail 958 | if: always() 959 | uses: ./.github/workflows/e2e/get_deployment_info 960 | with: 961 | expected_status_code: 404 962 | deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} 963 | organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} 964 | astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} 965 | astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} 966 | 967 | delete-test-deployments: 968 | name: Delete Deployment 969 | runs-on: ubuntu-latest 970 | needs: 971 | [ 972 | create-test-deployments, 973 | dag-deploy-test, 974 | pytests-test, 975 | custom-docker-image-test, 976 | dbt-deploy-test, 977 | deployment-preview-test, 978 | ] 979 | # ensure that we always try delete the deployment even if any of the tests fail 980 | if: always() 981 | steps: 982 | - name: Checkout code 983 | uses: actions/checkout@v4 984 | 985 | - name: Get Astro Environment Info 986 | id: get-astro-env-info 987 | uses: ./.github/workflows/e2e/get_astro_env_info 988 | with: 989 | input_workspace_id: ${{ github.event.inputs.workspace_id }} 990 | input_organization_id: ${{ github.event.inputs.org_id }} 991 | input_astronomer_host: ${{ github.event.inputs.astronomer_host }} 992 | input_astro_api_token: ${{ github.event.inputs.token }} 993 | secret_workspace_id: ${{ secrets.WORKSPACE_ID }} 994 | secret_organization_id: ${{ secrets.ORGANIZATION_ID }} 995 | secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} 996 | secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} 997 | 998 | - name: Install Astro CLI and set context 999 | run: | 1000 | curl -sSL https://install.astronomer.io | sudo bash -s 1001 | astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} 1002 | astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }} 1003 | 1004 | - name: Delete Deployment 1005 | run: | 1006 | astro deployment delete -f ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} 1007 | astro deployment delete -f ${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} 1008 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea/ 3 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | rules: 5 | line-length: disable 6 | document-start: 7 | present: false 8 | indentation: 9 | spaces: 2 10 | indent-sequences: true 11 | check-multi-line-strings: false # disabled because Bash scripts contain indented code 12 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @astronomer/dev-ex 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License") modified with 2 | Commons Clause Restriction; you may not use this code except in compliance with 3 | the License. You may obtain a copy of the License at: 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed 8 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 9 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 10 | specific language governing permissions and limitations under the License. 11 | 12 | "Commons Clause" License Condition v1.0 13 | 14 | The Software is provided to you by the Licensor under the License, as defined 15 | below, subject to the following condition. 16 | 17 | Without limiting other conditions in the License, the grant of rights under the 18 | License will not include, and the License does not grant to you, the right to 19 | Sell the Software. 20 | 21 | For purposes of the foregoing, "Sell" means practicing any or all of the rights 22 | granted to you under the License to provide to third parties, for a fee or other 23 | consideration (including without limitation fees for hosting or 24 | consulting/support services related to the Software), a product or service whose 25 | value derives, entirely or substantially, from the functionality of the 26 | Software. Any license notice or attribution required by the License must also 27 | include this Commons Clause License Condition notice. 28 | 29 | Software: Astronomer Github Actions 30 | License: Apache 2.0 31 | Licensor: Astronomer, Inc. 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deploy to Astro 2 | This GitHub action automates deploying code from your GitHub repository to a Deployment on [Astro](https://www.astronomer.io/product/), Astronomer's data orchestration platform and managed service for Apache Airflow. 3 | 4 | You can use and configure this GitHub action to easily deploy Apache Airflow DAGs to an Airflow environment on Astro. Specifically, you can: 5 | 6 | - Avoid manually running `astro deploy` with the Astro CLI every time you make a change to your Astro project. 7 | - Automate deploying code to Astro when you merge changes to a certain branch in your repository. 8 | - Incorporate unit tests for your DAGs as part of the deploy process. 9 | - Create/delete a Deployment Preview. A Deployment Preview is an Astro Deployment that mirrors the configuration of your original Deployment. 10 | 11 | This GitHub action runs as a step within a GitHub workflow file. When your CI/CD pipeline is triggered, this action: 12 | 13 | - Checks out your GitHub repository. 14 | - Optionally creates or deletes a Deployment Preview to test your code changes on before deploying to production. 15 | - Checks whether your commit only changed DAG code. 16 | - Optional. Tests DAG code with `pytest`. See [Run tests with pytest](https://docs.astronomer.io/astro/cli/test-your-astro-project-locally#run-tests-with-pytest). 17 | - Either runs: 18 | - `astro deploy --dags` if the commit only includes DAG code changes, 19 | - or `astro deploy` (as well as `astro dev parse`) if the commit includes _any_ non-DAG-code-related changes. 20 | 21 | ## Prerequisites 22 | 23 | To use this GitHub action, you need: 24 | 25 | - An Astro project. See [Create a project](https://docs.astronomer.io/astro/create-project). 26 | - A Deployment on Astro. See [Create a Deployment](https://docs.astronomer.io/astro/create-deployment). 27 | - An Organization, Workspace, or Deployment API Token. See [API Tokens](https://docs.astronomer.io/astro/workspace-api-tokens) 28 | - Or (deprecated) a Deployment API key ID and secret. See [Deployment API keys](https://docs.astronomer.io/astro/api-keys). 29 | 30 | > [!TIP] 31 | > Astronomer recommends using [GitHub Actions secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) to store `ASTRO_API_TOKEN` or Deployment API Keys. See the example in [Workflow file examples](https://github.com/astronomer/deploy-action#workflow-file-examples). 32 | 33 | ## Use this action 34 | 35 | To use this action, read [Automate code deploys with CI/CD](https://docs.astronomer.io/astro/ci-cd?tab=multiple%20branch#github-actions-dag-based-deploy). You will: 36 | 37 | 1. Create a GitHub Actions workflow in your repository that uses the latest version of this action. For example, `astronomer/deploy-action@v0.4`. 38 | 2. Configure the workflow to fit your team's use case. This could include creating a deployment preview or adding tests. See [Configuration options](https://github.com/astronomer/deploy-action#configuration-options). 39 | 3. Make changes to your Astro project files in GitHub and let this GitHub Actions workflow take care of deploying your code to Astro. 40 | 41 | > [!TIP] 42 | > Astronomer recommends setting up multiple environments on Astro. See the [Multiple branch GitHub Actions workflow](https://docs.astronomer.io/astro/ci-cd?tab=multibranch#github-actions-image-only-deploys) in Astronomer documentation. 43 | 44 | 45 | ## Configuration options 46 | 47 | The following table lists the configuration options for the Deploy to Astro action. 48 | 49 | | Name | Default | Description | 50 | | ---|---|--- | 51 | | `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `deploy`, `create-deployment-preview`, `delete-deployment-preview` or `deploy-deployment-preview`. If using `deploy` or `deploy-deployment-preview` one should also specify `deploy-type`. | 52 | | `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs or DBT project. Possible options are `infer`, `dags-only`, `image-and-dags` or `dbt`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. For description on each deploy type checkout [deploy type details](https://github.com/astronomer/deploy-action#deploy-type-details) | 53 | | `deployment-id` | `false` | Specifies the id of the deployment you to make a preview from or are deploying too. | 54 | | `deployment-name` | `false` | Specifies The name of the deployment you want to make preview from or are deploying too. Cannot be used with `deployment-id` | 55 | | `description` | | Configure a description for a deploy to Astro. Description will be visible in the Deploy History tab. | 56 | | `root-folder` | `.` | Path to the Astro project, or dbt project for dbt deploys. | 57 | | `parse` | `false` | When set to `true`, DAGs are parsed for errors before deploying to Astro. Note that when an image deploy is performed (i.e. `astro deploy`), parsing is also executed by default. Parsing is _not_ performed automatically for DAG-only deploys (i.e. `astro deploy --dags`). | 58 | | `pytest` | `false` | When set to `true`, all pytests in the `tests` directory of your Astro project are run before deploying to Astro. See [Run tests with pytest](https://docs.astronomer.io/astro/cli/test-your-astro-project-locally#run-tests-with-pytest) | 59 | | `pytest-file` | (all tests run) | Specifies a custom pytest file to run with the pytest command. For example, you could specify `/tests/test-tags.py`.| 60 | | `force` | `false` | When set to `true`, your code is deployed and skips any pytest or parsing errors. | 61 | | `image-name` | | Specifies a custom, locally built image to deploy. To be used with `deploy-type` set to `image-and-dags` or `infer` | 62 | | `workspace` | | Workspace id to select. Only required when `ASTRO_API_TOKEN` is given an organization token. | 63 | | `preview-name` | `false` | Specifies custom preview name. By default this is branch name “_” deployment name. | 64 | | `checkout` | `true` | Whether to checkout the repo as the first step. Set this to false if you want to modify repo contents before invoking the action. Your custom checkout step needs to have `fetch-depth` of `0` and `ref` equal to `${{ github.event.after }}` so all the commits in the PR are checked out. Look at the checkout step that runs within this action for reference. | 65 | | `deploy-image` | `false` | If true image and DAGs will deploy for any action that deploys code. NOTE: This option is deprecated and will be removed in a future release. Use `deploy-type: image-and-dags` instead. | 66 | | `build-secrets` | `` | Mimics docker build --secret flag. See https://docs.docker.com/build/building/secrets/ for more information. Example input 'id=mysecret,src=secrets.txt'. | 67 | | `mount-path` | `` | Path to mount dbt project in Airflow, for reference by DAGs. Default /usr/local/airflow/dbt/{dbt project name} | 68 | | `checkout-submodules` | `false` | Whether to checkout submodules when cloning the repository: `false` to disable (default), `true` to checkout submodules or `recursive` to recursively checkout submodules. Works only when `checkout` is set to `true`. Works only when `checkout` is set to `true`. | 69 | | `wake-on-deploy` | `false` | If true, the deployment will be woken up from hibernation before deploying. NOTE: This option overrides the deployment's hibernation override spec. | 70 | | `cli-version` | `` | The desired Astro CLI version to use. The latest version is used if left unset. | 71 | 72 | 73 | ## Outputs 74 | 75 | The following table lists the outputs for the Deploy to Astro action. 76 | 77 | | Name | Description | 78 | | ---|--- | 79 | | `preview-id` | The ID of the created deployment preview. Only works when action=create-deployment-preview. | 80 | 81 | ## Deploy Type Details 82 | 83 | The following section describe each of the deploy type input value in detail to avoid any confusion: 84 | 1. `infer`: In this mode, deploy-action would run through all the file changes: 85 | - if there are no file changes in the configured root-folder then it skips deploy 86 | - if there are changes only in `dags/` folder, then it will do a dags deploy 87 | - otherwise it would do a complete image deploy 88 | 89 | 2. `image-and-dags`: In this mode, deploy-action would run through all the file changes: 90 | - if there are no file changes in the configured root-folder then it skips deploy 91 | - otherwise it would do a complete image deploy 92 | 93 | 3. `dags-only`: In this mode, deploy-action would run through all the file changes: 94 | - if there are no file changes in the configured root-folder then it skips deploy 95 | - if all the file changes are in folders except `dags` folder then it skips deploy 96 | - otherwise it would do a dag only deploy 97 | 98 | 4. `dbt`: In this mode, deploy-action would run through all the file changes: 99 | - if there are no file changes in the configured root-folder then it skips deploy 100 | - otherwise it would do a dbt deploy 101 | 102 | 103 | ## Workflow file examples 104 | 105 | 106 | In the following example, the GitHub action deploys code to Astro. This example assumes that you have one Astro Deployment and one branch. When a change is merged to the `main` branch, your Astro project is deployed to Astro. DAG files are parsed on every deploy and no pytests are ran. 107 | 108 | ```yaml 109 | name: Astronomer CI - Deploy code 110 | 111 | on: 112 | push: 113 | branches: 114 | - main 115 | 116 | env: 117 | ## Set API Token as an environment variable 118 | ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} 119 | 120 | jobs: 121 | deploy: 122 | runs-on: ubuntu-latest 123 | steps: 124 | - name: Deploy to Astro 125 | uses: astronomer/deploy-action@v0.10.1 126 | with: 127 | deployment-id: 128 | parse: true 129 | ``` 130 | 131 | Use the following topics to further configure the action based on your needs. 132 | 133 | ### Change the Root folder 134 | 135 | In the following example, the folder `/example-dags/` is specified as the root folder. 136 | 137 | ```yaml 138 | steps: 139 | - name: Deploy to Astro 140 | uses: astronomer/deploy-action@v0.10.1 141 | with: 142 | deployment-id: 143 | root-folder: /example-dags/ 144 | ``` 145 | 146 | ### Run Pytests 147 | 148 | In the following example, the pytest located at `/tests/test-tags.py` runs before deploying to Astro. 149 | 150 | ```yaml 151 | steps: 152 | - name: Deploy to Astro 153 | uses: astronomer/deploy-action@v0.10.1 154 | with: 155 | deployment-id: 156 | pytest: true 157 | pytest-file: /tests/test-tags.py 158 | ``` 159 | 160 | ### Ignore parsing and testing 161 | 162 | In the following example, `force` is enabled and both the DAG parse and pytest processes are skipped. 163 | 164 | ```yaml 165 | steps: 166 | - name: Deploy to Astro 167 | uses: astronomer/deploy-action@v0.10.1 168 | with: 169 | deployment-id: 170 | force: true 171 | ``` 172 | 173 | ### Deploy a custom Docker image 174 | 175 | In the following example, a custom Docker image is built and deployed to an Astro Deployment. 176 | 177 | ```yaml 178 | name: Astronomer CI - Additional build-time args 179 | 180 | on: 181 | push: 182 | branches: 183 | - main 184 | 185 | jobs: 186 | build: 187 | runs-on: ubuntu-latest 188 | env: 189 | ## Set API Token as an environment variable 190 | ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} 191 | steps: 192 | - name: Check out the repo 193 | uses: actions/checkout@v3 194 | - name: Create image tag 195 | id: image_tag 196 | run: echo ::set-output name=image_tag::astro-$(date +%Y%m%d%H%M%S) 197 | - name: Build image 198 | uses: docker/build-push-action@v2 199 | with: 200 | tags: ${{ steps.image_tag.outputs.image_tag }} 201 | load: true 202 | # Define your custom image's build arguments, contexts, and connections here using 203 | # the available GitHub Action settings: 204 | # https://github.com/docker/build-push-action#customizing . 205 | # This example uses `build-args` , but your use case might require configuring 206 | # different values. 207 | build-args: | 208 | 209 | - name: Deploy to Astro 210 | uses: astronomer/deploy-action@v0.10.1 211 | with: 212 | deployment-id: 213 | deploy-type: image-and-dags 214 | image-name: ${{ steps.image_tag.outputs.image_tag }} 215 | 216 | ``` 217 | 218 | ### Deploy a DBT project 219 | 220 | In the following example we would be deploying the dbt project located at `dbt` folder in the Github repo 221 | 222 | ```yaml 223 | steps: 224 | - name: DBT Deploy to Astro 225 | uses: astronomer/deploy-action@v0.10.1 226 | with: 227 | deployment-id: 228 | deploy-type: dbt 229 | root-folder: dbt 230 | ``` 231 | 232 | ### Deploy DAGs and DBT project from same repo 233 | 234 | In the following example we would setup a workflow to deploy dags/images located at `astro-project` and dbt deploy from dbt project located at `dbt` folder in the same Github repo 235 | 236 | ```yaml 237 | steps: 238 | - name: DBT Deploy to Astro 239 | uses: astronomer/deploy-action@v0.10.1 240 | with: 241 | deployment-id: 242 | deploy-type: dbt 243 | root-folder: dbt 244 | - name: DAGs/Image Deploy to Astro 245 | uses: astronomer/deploy-action@v0.10.0 246 | with: 247 | deployment-id: 248 | root-folder: astro-project/ 249 | parse: true 250 | ``` 251 | 252 | ### Wake on deploy 253 | 254 | In the following example, the deployment is woken up from hibernation before deploying via the `wake-on-deploy` option. 255 | 256 | ```yaml 257 | steps: 258 | - name: Deploy to Astro 259 | uses: astronomer/deploy-action@v0.10.1 260 | with: 261 | deployment-id: 262 | wake-on-deploy: true 263 | ``` 264 | 265 | This is especially useful when you have a deployment that is hibernated and you want to deploy to it. This option overrides the deployment's hibernation override spec. 266 | 267 | ## Deployment Preview Templates 268 | 269 | This section contains four workflow files that you will need in your repository to have a full Deployment Preview Cycle running for your Deployment. A Deployment Preview is an Astro Deployment that mirrors the configuration of your original Deployment. This Deployment Preview can be used to test your new pipelines changes before pushing them to your original Deployment. The scripts below will take your pipeline changes through the following flow: 270 | 271 | 1. When a new branch is created a Deployment Preview will be created based off your original Deployment 272 | 2. When a PR is created from a branch code changes will be deployed to the Deployment Preview 273 | 3. When a PR is merged into your "main" branch code changes will be deployed to the original Deployment 274 | 4. When a branch is deleted the corresponding Deployment Preview will also be deleted 275 | 276 | ## Create Deployment Preview 277 | 278 | ```yaml 279 | name: Astronomer CI - Create deployment preview 280 | 281 | on: 282 | create: 283 | branches: 284 | - "**" 285 | 286 | env: 287 | ## Sets Deployment API key credentials as environment variables 288 | ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} 289 | 290 | jobs: 291 | deploy: 292 | runs-on: ubuntu-latest 293 | steps: 294 | - name: Create Deployment Preview 295 | uses: astronomer/deploy-action@v0.10.1 296 | with: 297 | action: create-deployment-preview 298 | deployment-id: 299 | ``` 300 | 301 | ## Deploy to Deployment Preview 302 | 303 | ```yaml 304 | name: Astronomer CI - Deploy code to Preview 305 | 306 | on: 307 | pull_request: 308 | branches: 309 | - main 310 | 311 | env: 312 | ## Sets Deployment API key credentials as environment variables 313 | ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} 314 | 315 | jobs: 316 | deploy: 317 | runs-on: ubuntu-latest 318 | steps: 319 | - name: Deploy to Deployment Preview 320 | uses: astronomer/deploy-action@v0.10.1 321 | with: 322 | action: deploy-deployment-preview 323 | deployment-id: 324 | ``` 325 | 326 | ## DBT Deploy to Deployment Preview 327 | 328 | ```yaml 329 | name: Astronomer - DBT Deploy code to Preview 330 | 331 | on: 332 | pull_request: 333 | branches: 334 | - main 335 | 336 | env: 337 | ## Sets Deployment API key credentials as environment variables 338 | ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} 339 | 340 | jobs: 341 | deploy: 342 | runs-on: ubuntu-latest 343 | steps: 344 | - name: Deploy to Deployment Preview 345 | uses: astronomer/deploy-action@v0.10.1 346 | with: 347 | action: deploy-deployment-preview 348 | deploy-type: dbt 349 | deployment-id: 350 | root-folder: dbt 351 | ``` 352 | 353 | ## Delete Deployment Preview 354 | 355 | ```yaml 356 | name: Astronomer CI - Delete Deployment Preview 357 | 358 | on: 359 | delete: 360 | branches: 361 | - "**" 362 | 363 | env: 364 | ## Sets Deployment API key credentials as environment variables 365 | ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} 366 | 367 | jobs: 368 | deploy: 369 | runs-on: ubuntu-latest 370 | steps: 371 | - name: Delete Deployment Preview 372 | uses: astronomer/deploy-action@v0.10.1 373 | with: 374 | action: delete-deployment-preview 375 | deployment-id: 376 | ``` 377 | 378 | ## Deploy to Original Deployment 379 | 380 | ```yaml 381 | name: Astronomer CI - Deploy code to Astro 382 | 383 | on: 384 | push: 385 | branches: 386 | - main 387 | 388 | env: 389 | ## Sets Deployment API key credentials as environment variables 390 | ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} 391 | 392 | jobs: 393 | deploy: 394 | runs-on: ubuntu-latest 395 | steps: 396 | - name: Deploy to Astro 397 | uses: astronomer/deploy-action@v0.10.1 398 | with: 399 | deployment-id: 400 | ``` 401 | -------------------------------------------------------------------------------- /action.yaml: -------------------------------------------------------------------------------- 1 | name: "Deploy Apache Airflow DAGs to Astro" 2 | description: "Test your DAGs and deploy your Astro project to a Deployment on Astro, Astronomer's managed Airflow service." 3 | author: "Astronomer" 4 | branding: 5 | icon: "upload-cloud" 6 | color: "purple" 7 | inputs: 8 | root-folder: 9 | required: false 10 | default: "" 11 | description: "Path to the Astro project, or dbt project for dbt deploys." 12 | parse: 13 | required: false 14 | default: false 15 | description: "If true DAGs will be parsed before deploying to Astro." 16 | pytest: 17 | required: false 18 | default: false 19 | description: "if true custom pytests will be ran before deploying to Astro." 20 | pytest-file: 21 | required: false 22 | default: tests/ 23 | description: "Specify custom pytest files to run with the pytest command." 24 | force: 25 | required: false 26 | default: false 27 | description: "If true your code will be force deployed to Astronomer. Mostly used to skip parse test on image deploys." 28 | image-name: 29 | required: false 30 | default: no-custom-image 31 | description: "Specify a custom built image to deploy to an Asto Deployment. To be used with 'deploy-type' set to 'image-and-dags' or 'infer'" 32 | action: 33 | required: false 34 | default: deploy 35 | description: "Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either 'deploy', 'create-deployment-preview', 'delete-deployment-preview', or 'deploy-deployment-preview'. If using 'deploy' or 'deploy-deployment-preview' one should also specify 'deploy-type'." 36 | deployment-name: 37 | required: false 38 | description: "The name of the Deployment you want to make preview from or are deploying to." 39 | deployment-id: 40 | required: false 41 | description: "The id of the Deployment you to make a preview from or are deploying to." 42 | workspace: 43 | required: false 44 | description: "Workspace id to select. Only required when `ASTRO_API_TOKEN` is given an organization token." 45 | preview-name: 46 | required: false 47 | description: "Custom preview name. By default this is _." 48 | copy-connections: 49 | required: false 50 | default: true 51 | description: "Copy connections from the original Deployment to the new deployment preview." 52 | copy-airflow-variables: 53 | required: false 54 | default: true 55 | description: "Copy Airflow variables from the original Deployment to the new deployment preview." 56 | copy-pools: 57 | required: false 58 | default: true 59 | description: "Copy pools from the original Deployment to the new deployment preview." 60 | cli-version: 61 | required: false 62 | default: "" 63 | description: "The desired Astro CLI version to use" 64 | checkout: 65 | required: false 66 | default: true 67 | description: "Whether to checkout the repo as the first step. Set this to false if you want to modify repo contents before invoking the action" 68 | description: 69 | required: false 70 | description: > 71 | A description to set for deploying to Astro. This is equivalent to running `astro deploy --description "..."` with the Astro CLI. 72 | The description is visible in the Deploy History tab in your Astro Deployment, and can be helpful to explain what triggered a deploy. 73 | 74 | For example, to display the most recent commit that resulted in an Astro deploy, you could configure 75 | `description: "Deployed from commit ..."` with the value of `github.event.after`. 76 | This would display e.g. "Deployed from commit da39a3ee5e6b4b0d3255bfef95601890afd80709". 77 | Reference: https://docs.github.com/en/webhooks/webhook-events-and-payloads#push. 78 | deploy-image: 79 | required: false 80 | default: false 81 | description: "If true image and DAGs will deploy. NOTE: This option is deprecated and will be removed in a future release. Use `deploy-type: image-and-dags` instead." 82 | deploy-type: 83 | required: false 84 | default: "infer" 85 | description: "Specify the type of deploy you would like to do. Use this option to deploy only DAGs or the image or DBT project. Specify either 'infer', 'dags-only', 'image-and-dags', or 'dbt'. 'infer' option would infer between DAG only deploy and image and DAG deploy based on updated files." 86 | build-secrets: 87 | required: false 88 | description: "Mimics docker build --secret flag. See https://docs.docker.com/build/building/secrets/ for more information. Example input 'id=mysecret,src=secrets.txt'" 89 | mount-path: 90 | required: false 91 | default: "" 92 | description: "Path to mount dbt project in Airflow, for reference by DAGs. Default /usr/local/airflow/dbt/{dbt project name}" 93 | checkout-submodules: 94 | required: false 95 | default: false 96 | description: "Whether to checkout submodules when cloning the repository: `false` to disable (default), `true` to checkout submodules or `recursive` to recursively checkout submodules. Works only when `checkout` is set to `true`." 97 | wake-on-deploy: 98 | required: false 99 | default: false 100 | description: "If true, the deployment will be woken up from hibernation before deploying. NOTE: This option overrides the deployment's hibernation override spec." 101 | outputs: 102 | preview-id: 103 | description: "The ID of the created deployment preview. Only works when action=create-deployment-preview" 104 | value: ${{ steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID }} 105 | 106 | runs: 107 | using: "composite" 108 | steps: 109 | - name: checkout repo 110 | uses: actions/checkout@v4 111 | if: inputs.checkout == 'true' 112 | with: 113 | fetch-depth: 0 # Fetch all history 114 | ref: ${{ github.event.after }} 115 | clean: false 116 | submodules: ${{ inputs.checkout-submodules }} 117 | - name: Warn about deprecated deploy-image option 118 | if: inputs.deploy-image == true 119 | shell: bash 120 | run: | 121 | echo "The 'deploy-image' option is deprecated and will be removed in 1.0 release. Use 'deploy-type: image-and-dags' instead." 122 | - name: Install Astro CLI 123 | run: | 124 | 125 | echo ::group::Install Astro CLI 126 | CLI_VERSION=${{ inputs.cli-version }} 127 | # if Astro CLI is pre-installed, fetch it's version 128 | if command -v astro &> /dev/null; then 129 | # "astro version" commands returns the CLI version, its output is of the form: "Astro CLI Version: 1.29.0" 130 | CLI_VERSION=$(astro version | awk '{print $4}') 131 | fi 132 | 133 | # Check if the Astro CLI version is less than 1.28.1 for dbt-deploy 134 | if [[ "${{ inputs.deploy-type }}" == "dbt" && $CLI_VERSION != "" ]]; then 135 | #install semver to compare versions 136 | npm install -g semver 137 | 138 | REQUIRED_VERSION="1.28.1" 139 | CURRENT_VERSION=$(astro version | awk '{print $4}') 140 | if ! semver -r ">${REQUIRED_VERSION}" "${CURRENT_VERSION}"; then 141 | echo "DBT Deploy requires Astro CLI version $REQUIRED_VERSION or higher" 142 | exit 1 143 | fi 144 | fi 145 | 146 | # skip Astro CLI installation if already installed 147 | if command -v astro &> /dev/null; then 148 | echo "Astro CLI is already installed" 149 | exit 0 150 | fi 151 | 152 | # check if CLI_VERSION does not starts with "v", then add "v" to it 153 | if [[ $CLI_VERSION != "" && $CLI_VERSION != v* ]]; then 154 | CLI_VERSION=v$CLI_VERSION 155 | fi 156 | 157 | if [[ $CLI_VERSION == "" ]]; then 158 | curl -sSL https://install.astronomer.io | sudo bash -s 159 | else 160 | curl -sSL https://install.astronomer.io | sudo bash -s -- $CLI_VERSION 161 | fi 162 | echo ::endgroup:: 163 | shell: bash 164 | - name: Determine Deploy Deployment 165 | run: | 166 | 167 | echo ::group::Determine Deploy Deployment 168 | 169 | # validate action input 170 | if [[ ${{ inputs.action }} != create-deployment-preview && ${{ inputs.action }} != delete-deployment-preview && ${{ inputs.action }} != deploy-deployment-preview && ${{ inputs.action }} != deploy ]]; then 171 | echo ERROR: you specified an improper action input. Action must be deploy, deploy-deployment-preview, create-deployment-preview, or delete-deployment-preview. 172 | exit 1 # terminate and indicate error 173 | fi 174 | 175 | # Select workspace if specified 176 | if [[ "${{ inputs.workspace }}" != "" ]]; then 177 | astro workspace switch ${{ inputs.workspace }} 178 | fi 179 | # error if both deployment name and id are used 180 | if [[ "${{ inputs.deployment-name }}" != "" && "${{ inputs.deployment-id }}" != "" ]]; then 181 | echo ERROR: cannot specify both a Deployment ID and Name 182 | exit 1 # terminate and indicate error 183 | fi 184 | # figure out deployment id 185 | if [[ "${{ inputs.deployment-name }}" != "" ]]; then 186 | # get deployment-id 187 | DEPLOYMENT_ID="$(astro deployment inspect --clean-output -n "${{ inputs.deployment-name }}" --key metadata.deployment_id)" 188 | fi 189 | 190 | if [[ "${{ inputs.deployment-id }}" != "" ]]; then 191 | DEPLOYMENT_ID="${{ inputs.deployment-id }}" 192 | fi 193 | # create deployment preview if action is create-deployment-preview 194 | if [[ ${{ inputs.action }} == create-deployment-preview ]]; then 195 | 196 | if [[ "${{ inputs.deployment-name }}" == "" && "${{ inputs.deployment-id }}" == "" ]]; then 197 | echo ERROR: cannot create a deployment preview without specifying a deployment name or id 198 | exit 1 # terminate and indicate error 199 | fi 200 | 201 | if [[ "${{ inputs.preview-name }}" != "" ]]; then 202 | BRANCH_DEPLOYMENT_NAME="${{ inputs.preview-name }}" 203 | else 204 | # get branch name 205 | DEPLOYMENT_NAME="$(astro deployment inspect $DEPLOYMENT_ID --clean-output --key configuration.name)" 206 | if [[ ${GITHUB_HEAD_REF##*/} != "" ]]; then 207 | BRANCH_DEPLOYMENT_NAME=${GITHUB_HEAD_REF##*/}_$DEPLOYMENT_NAME 208 | else 209 | branch_ref="${{ github.ref }}" 210 | branch_name="${branch_ref##*/}" 211 | BRANCH_DEPLOYMENT_NAME=${branch_name}_$DEPLOYMENT_NAME 212 | fi 213 | BRANCH_DEPLOYMENT_NAME="${BRANCH_DEPLOYMENT_NAME// /_}" 214 | echo $BRANCH_DEPLOYMENT_NAME 215 | fi 216 | 217 | # Create template of deployment to be copied with 218 | astro deployment inspect $DEPLOYMENT_ID --clean-output --template > deployment-preview-template.yaml # automatically creates deployment-preview-template.yaml file 219 | 220 | # Add name to deployment template file 221 | sed -i "s| name:.*| name: ${BRANCH_DEPLOYMENT_NAME}|g" deployment-preview-template.yaml 222 | 223 | # Create new deployment preview based on the deployment template file 224 | astro deployment create --deployment-file deployment-preview-template.yaml 225 | 226 | # TODO: we need to add wait for deployment to be created, otherwise operation to copy airflow resources like connection is flaky if webserver is not up by then 227 | 228 | # Get final Deployment ID 229 | echo "FINAL_DEPLOYMENT_ID=$(astro deployment inspect --clean-output -n $BRANCH_DEPLOYMENT_NAME --key metadata.deployment_id)" >> $GITHUB_OUTPUT 230 | 231 | # Get original Deployment ID 232 | echo "ORIGINAL_DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT 233 | fi 234 | 235 | # delete deployment preview and skip deploy if action is delete-deployment-preview 236 | if [[ ${{ inputs.action }} == delete-deployment-preview ]]; then 237 | 238 | if [[ "${{ inputs.deployment-name }}" == "" && "${{ inputs.deployment-id }}" == "" ]]; then 239 | echo ERROR: cannot delete a deployment preview without specifying a deployment name or id 240 | exit 1 # terminate and indicate error 241 | fi 242 | 243 | if [[ "${{ inputs.preview-name }}" != "" ]]; then 244 | BRANCH_DEPLOYMENT_NAME="${{ inputs.preview-name }}" 245 | else 246 | # Get deployment name from Astro if preview-name is not provided 247 | DEPLOYMENT_NAME="$(astro deployment inspect $DEPLOYMENT_ID --clean-output --key configuration.name)" 248 | # using github.event.ref to refer the actual branch getting deleted 249 | branch_ref="${{ github.event.ref }}" 250 | branch_name="${branch_ref##*/}" 251 | BRANCH_DEPLOYMENT_NAME=${branch_name}_$DEPLOYMENT_NAME 252 | BRANCH_DEPLOYMENT_NAME="${BRANCH_DEPLOYMENT_NAME// /_}" 253 | fi 254 | # delete branch deployment 255 | astro deployment delete -n $BRANCH_DEPLOYMENT_NAME -f 256 | fi 257 | 258 | # # deploy to deployment preview if action is deploy-deployment-preview 259 | if [[ ${{ inputs.action }} == deploy-deployment-preview ]]; then 260 | 261 | if [[ "${{ inputs.deployment-name }}" == "" && "${{ inputs.deployment-id }}" == "" ]]; then 262 | echo ERROR: cannot deploy to a deployment preview without specifying a deployment name or id 263 | exit 1 # terminate and indicate error 264 | fi 265 | 266 | if [[ "${{ inputs.preview-name }}" != "" ]]; then 267 | BRANCH_DEPLOYMENT_NAME="${{ inputs.preview-name }}" 268 | else 269 | DEPLOYMENT_NAME="$(astro deployment inspect $DEPLOYMENT_ID --clean-output --key configuration.name)" 270 | if [[ ${GITHUB_HEAD_REF##*/} != "" ]]; then 271 | BRANCH_DEPLOYMENT_NAME=${GITHUB_HEAD_REF##*/}_$DEPLOYMENT_NAME 272 | else 273 | # BRANCH_DEPLOYMENT_NAME=${{ github.event.ref }}_$DEPLOYMENT_NAME 274 | branch_ref="${{ github.ref }}" 275 | branch_name="${branch_ref##*/}" 276 | BRANCH_DEPLOYMENT_NAME=${branch_name}_$DEPLOYMENT_NAME 277 | fi 278 | BRANCH_DEPLOYMENT_NAME="${BRANCH_DEPLOYMENT_NAME// /_}" 279 | fi 280 | 281 | # Get Deployment ID 282 | echo "FINAL_DEPLOYMENT_ID=$(astro deployment inspect --clean-output -n $BRANCH_DEPLOYMENT_NAME --key metadata.deployment_id)" >> $GITHUB_OUTPUT 283 | fi 284 | 285 | # if action is deploy or dbt deploy simply set final Deployment id to Deployment id to streamline the deployment id reference in rest of the workflow 286 | if [[ ${{ inputs.action }} == deploy ]]; then 287 | echo "FINAL_DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT 288 | fi 289 | 290 | echo ::endgroup:: 291 | shell: bash 292 | id: deployment-preview 293 | - name: Determine if DAG Deploy is enabled 294 | run: | 295 | 296 | echo ::group::Determine if DAG Deploy is enabled 297 | if [[ ${{ inputs.action }} != create-deployment-preview && ${{ inputs.action }} != delete-deployment-preview ]]; then 298 | echo "DAG_DEPLOY_ENABLED=$(astro deployment inspect ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --clean-output --key configuration.dag_deploy_enabled)" >> $GITHUB_OUTPUT 299 | else 300 | echo "DAG_DEPLOY_ENABLED=false" >> $GITHUB_OUTPUT 301 | fi 302 | echo ::endgroup:: 303 | shell: bash 304 | id: dag-deploy-enabled 305 | - name: Get DBT Deploy Options 306 | if: ${{ inputs.deploy-type == 'dbt' }} 307 | run: | 308 | 309 | echo ::group::Get DBT Deploy Options 310 | 311 | if [[ "${{ inputs.root-folder }}" != "" ]]; then 312 | cd ${{ inputs.root-folder }} 313 | fi 314 | 315 | branch=$(echo "${GITHUB_REF#refs/heads/}") 316 | echo "Branch pushed to: $branch" 317 | git fetch origin $branch 318 | 319 | DBT_DEPLOY=false 320 | 321 | # case when the triggered event is a manual workflow dispatch or a new branch or tag creation, we would need to deploy the dbt project because we cannot determine that it does not need to be deployed 322 | GITHUB_EVENT_BEFORE=${{ github.event.before }} 323 | GITHUB_EVENT_AFTER=${{ github.event.after }} 324 | if [[ "$GITHUB_EVENT_BEFORE" == "0000000000000000000000000000000000000000" || -z $GITHUB_EVENT_BEFORE && -z $GITHUB_EVENT_AFTER ]]; then 325 | DBT_DEPLOY=true 326 | files=() 327 | else 328 | files=$(git diff --name-only $GITHUB_EVENT_BEFORE $GITHUB_EVENT_AFTER) 329 | echo "files changed: $files" 330 | fi 331 | 332 | for file in $files; do 333 | if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then 334 | echo $file is part of configured root folder, so would be triggering a dbt deploy 335 | DBT_DEPLOY=true 336 | fi 337 | done 338 | 339 | options="" 340 | if [[ $DBT_DEPLOY == true ]]; then 341 | # Add mount path option 342 | if [[ "${{ inputs.mount-path }}" != "" ]]; then 343 | options="$options --mount-path=${{ inputs.mount-path }}" 344 | fi 345 | 346 | # Add description option 347 | if [[ "${{ inputs.description }}" != "" ]]; then 348 | options="$options --description '${{ inputs.description }}'" 349 | fi 350 | fi 351 | 352 | echo "DBT_DEPLOY=$DBT_DEPLOY" >> $GITHUB_OUTPUT 353 | echo "DBT_OPTIONS=$options" >> $GITHUB_OUTPUT 354 | echo ::endgroup:: 355 | shell: bash 356 | id: dbt-deploy-options 357 | - name: Get Deploy Type 358 | if: ${{ inputs.deploy-type == 'infer' || inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'image-and-dags' }} 359 | run: | 360 | 361 | # infer based on files changed to deploy only dags or image and dags 362 | echo ::group::Get Deploy Type 363 | 364 | if [[ "${{ inputs.root-folder }}" != "" ]]; then 365 | cd ${{ inputs.root-folder }} 366 | fi 367 | 368 | DAGS_ONLY_DEPLOY=false 369 | SKIP_IMAGE_OR_DAGS_DEPLOY=false 370 | files=() 371 | 372 | GITHUB_EVENT_BEFORE=${{ github.event.before }} 373 | GITHUB_EVENT_AFTER=${{ github.event.after }} 374 | # case when the triggered event is a manual workflow dispatch or a new branch or tag creation, we would need to deploy the image because we cannot determine that it does not need to be deployed 375 | if [[ "$GITHUB_EVENT_BEFORE" == "0000000000000000000000000000000000000000" || -z $GITHUB_EVENT_BEFORE && -z $GITHUB_EVENT_AFTER ]]; then 376 | echo "Manual workflow dispatch or a new branch or tag creation, hence missing github event before and/or after commit hash" 377 | else 378 | echo "event that triggered the workflow: $GITHUB_REF" 379 | branch=$(echo "${GITHUB_REF#refs/heads/}") 380 | git fetch origin $branch 381 | if ! git cat-file -e "${{ github.event.before }}" 2>/dev/null; then 382 | echo "Commit ${{ github.event.before }} does not exist, falling back to image deploy." 383 | else 384 | SKIP_IMAGE_OR_DAGS_DEPLOY=true 385 | files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }}) 386 | echo "files changed: $files" 387 | fi 388 | fi 389 | 390 | for file in $files; do 391 | if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then 392 | echo $file is part of the input root folder 393 | SKIP_IMAGE_OR_DAGS_DEPLOY=false 394 | if [[ ${{ inputs.deploy-type }} == 'infer' ]]; then 395 | if [[ $file == *"dags/"* ]]; then 396 | echo $file is part of dags folder 397 | DAGS_ONLY_DEPLOY=true 398 | else 399 | DAGS_ONLY_DEPLOY=false 400 | break 401 | fi 402 | elif [[ ${{ inputs.deploy-type }} == 'dags-only' ]]; then 403 | if [[ $file == *"dags/"* ]]; then 404 | echo $file is part of dags folder 405 | DAGS_ONLY_DEPLOY=true 406 | fi 407 | elif [[ ${{ inputs.deploy-type }} == 'image-and-dags' ]]; then 408 | DAGS_ONLY_DEPLOY=false 409 | break 410 | fi 411 | fi 412 | done 413 | 414 | # Note: the order of these following checks is important to ensure that we skip/trigger deploy correctly in following cases: 415 | # 1. When there is no change in the input root folder we should skip deploy, but not when it's deployment preview create action 416 | # 2. When user has passed a custom image then we would need to deploy the image 417 | # 3. When the action is deployment preview delete, then we should skip any form of deploy 418 | if [[ $SKIP_IMAGE_OR_DAGS_DEPLOY == true ]]; then 419 | # skip all deploy steps 420 | DAGS_ONLY_DEPLOY=false 421 | fi 422 | 423 | # check if user has passed a custom image or the action has created a new deployment preview, then we would need to deploy the image 424 | if [[ ${{ inputs.image-name }} != "no-custom-image" || ${{ steps.dag-deploy-enabled.outputs.DAG_DEPLOY_ENABLED }} == false || ${{ inputs.action }} == create-deployment-preview ]]; then 425 | SKIP_IMAGE_OR_DAGS_DEPLOY=false 426 | DAGS_ONLY_DEPLOY=false 427 | fi 428 | 429 | if [[ ${{ inputs.action }} == delete-deployment-preview ]]; then 430 | # skip all deploy steps 431 | SKIP_IMAGE_OR_DAGS_DEPLOY=true 432 | DAGS_ONLY_DEPLOY=false 433 | fi 434 | 435 | echo "DAGS_ONLY_DEPLOY=$DAGS_ONLY_DEPLOY" >> $GITHUB_OUTPUT 436 | echo "SKIP_IMAGE_OR_DAGS_DEPLOY=$SKIP_IMAGE_OR_DAGS_DEPLOY" >> $GITHUB_OUTPUT 437 | echo ::endgroup:: 438 | shell: bash 439 | id: deploy-type 440 | # If only DAGs changed and dag deploys is enabled, do a DAG-only deploy 441 | - name: Setup image or dags deploy options 442 | if: ${{ inputs.deploy-type == 'infer' || inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'image-and-dags' }} 443 | run: | 444 | 445 | echo ::group::Setup image or dags deploy options 446 | options="" 447 | 448 | # add parse option 449 | if [[ ${{ inputs.parse }} == true ]]; then 450 | options="--parse" 451 | fi 452 | 453 | # add pytest option 454 | if [[ ${{ inputs.pytest }} == true ]]; then 455 | options="$options --pytest --test ${{ inputs.pytest-file }}" 456 | fi 457 | 458 | # add custom image option 459 | if [[ ${{ inputs.image-name }} != no-custom-image && inputs.deploy-type != "dags-only" ]]; then 460 | options="$options --image-name ${{ inputs.image-name }}" 461 | fi 462 | 463 | # add force option 464 | if [[ ${{ inputs.force }} == true ]]; then 465 | options="$options --force" 466 | fi 467 | 468 | # Add description option 469 | if [[ "${{ inputs.description }}" != '' ]]; then 470 | options="$options --description '${{ inputs.description }}'" 471 | fi 472 | 473 | # Add build-secrets option 474 | if [[ "${{ inputs.build-secrets }}" != '' ]]; then 475 | options="$options --build-secrets '${{ inputs.build-secrets }}'" 476 | fi 477 | 478 | echo "OPTIONS=$options" >> $GITHUB_OUTPUT 479 | echo ::endgroup:: 480 | shell: bash 481 | id: deploy-options 482 | - name: Determine if Development Mode is enabled 483 | if: ${{ inputs.wake-on-deploy == 'true' && inputs.action != 'delete-deployment-preview' }} 484 | run: | 485 | echo ::group::Determine if Development Mode is enabled 486 | DEPLOYMENT_TYPE=$(astro deployment inspect ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --clean-output --key configuration.deployment_type) 487 | # only inspect development mode if deployment is not hybrid 488 | if [[ $DEPLOYMENT_TYPE != "HYBRID" ]]; then 489 | DEV_MODE=$(astro deployment inspect ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --clean-output --key configuration.is_development_mode) 490 | echo "Deployment development mode: $DEV_MODE" 491 | if [[ $DEV_MODE == "" ]]; then 492 | echo "DEVELOPMENT_MODE=false" >> $GITHUB_OUTPUT 493 | else 494 | echo "DEVELOPMENT_MODE=$DEV_MODE" >> $GITHUB_OUTPUT 495 | fi 496 | else 497 | echo "DEVELOPMENT_MODE=false" >> $GITHUB_OUTPUT 498 | fi 499 | echo ::endgroup:: 500 | shell: bash 501 | id: development-mode 502 | - name: Override to wake up the Deployment 503 | if: ${{ inputs.wake-on-deploy == 'true' && inputs.action != 'delete-deployment-preview' && steps.development-mode.outputs.DEVELOPMENT_MODE == 'true' }} 504 | run: | 505 | echo ::group::Override to wake up the Deployment 506 | astro deployment wake-up ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --force 507 | # Wait for the deployment to wake up 508 | while [ "$(astro deployment inspect ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --clean-output --key metadata.status)" == "HIBERNATING" ] 509 | do 510 | sleep 5 511 | done 512 | echo ::endgroup:: 513 | shell: bash 514 | - name: DAG Deploy to Astro 515 | if: ${{ (inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'infer') && steps.deploy-type.outputs.DAGS_ONLY_DEPLOY == 'true' }} 516 | run: | 517 | 518 | echo ::group::DAG Deploy to Astro 519 | # Deploy only dags 520 | 521 | if [[ "${{ inputs.root-folder }}" != "" ]]; then 522 | cd ${{ inputs.root-folder }} 523 | fi 524 | 525 | astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --dags ${{steps.deploy-options.outputs.OPTIONS}} 526 | echo ::endgroup:: 527 | shell: bash 528 | # If any other files changed or dag deploys is disabled, deploy the entire Astro project 529 | - name: Image and DAG Deploy to Astro 530 | if: ${{ inputs.deploy-image == true || ( (inputs.deploy-type == 'image-and-dags' || inputs.deploy-type == 'infer') && steps.deploy-type.outputs.DAGS_ONLY_DEPLOY == 'false' && steps.deploy-type.outputs.SKIP_IMAGE_OR_DAGS_DEPLOY == 'false' ) }} 531 | run: | 532 | echo ::group::Image and DAG Deploy to Astro 533 | # Deploy image and DAGs 534 | 535 | if [[ "${{ inputs.root-folder }}" != "" ]]; then 536 | cd ${{ inputs.root-folder }} 537 | fi 538 | 539 | astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.deploy-options.outputs.OPTIONS}} 540 | echo ::endgroup:: 541 | shell: bash 542 | - name: DBT Deploy to Astro 543 | if: ${{ inputs.deploy-type == 'dbt' && steps.dbt-deploy-options.outputs.DBT_DEPLOY == 'true' }} 544 | run: | 545 | 546 | echo ::group::DBT Deploy to Astro 547 | 548 | if [[ "${{ inputs.root-folder }}" != "" ]]; then 549 | cd ${{ inputs.root-folder }} 550 | fi 551 | 552 | astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.dbt-deploy-options.outputs.DBT_OPTIONS}} 553 | echo ::endgroup:: 554 | shell: bash 555 | - name: Copy Airflow Connections, Variables, and Pools 556 | if: ${{ inputs.action == 'create-deployment-preview' }} 557 | run: | 558 | 559 | echo ::group::Copy Airflow Connections, Variables, and Pools 560 | if [[ ${{ inputs.copy-connections }} == true ]]; then 561 | astro deployment connection copy --source-id ${{steps.deployment-preview.outputs.ORIGINAL_DEPLOYMENT_ID}} --target-id ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} 562 | fi 563 | if [[ ${{ inputs.copy-airflow-variables }} == true ]]; then 564 | astro deployment airflow-variable copy --source-id ${{steps.deployment-preview.outputs.ORIGINAL_DEPLOYMENT_ID}} --target-id ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} 565 | fi 566 | if [[ ${{ inputs.copy-pools }} == true ]]; then 567 | astro deployment pool copy --source-id ${{steps.deployment-preview.outputs.ORIGINAL_DEPLOYMENT_ID}} --target-id ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} 568 | fi 569 | echo ::endgroup:: 570 | shell: bash 571 | - name: Remove override on Deployment to resume schedule 572 | if: ${{ inputs.wake-on-deploy == 'true' && inputs.action != 'delete-deployment-preview' && steps.development-mode.outputs.DEVELOPMENT_MODE == 'true' }} 573 | run: | 574 | echo ::group::Remove override on Deployment to resume schedule 575 | astro deployment wake-up ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --remove-override --force 576 | echo ::endgroup:: 577 | shell: bash 578 | -------------------------------------------------------------------------------- /e2e-setup/README.md: -------------------------------------------------------------------------------- 1 | # End-to-End Setup 2 | 3 | This folder contains the necessary setup for running end-to-end tests for the Astro Deploy Action. Below is the folder structure and a brief description of the contents at the first level. 4 | 5 | The E2E tests could be triggered automatically when a new commit is pushed to main, or it could be manually triggered. E2E setup also supports specifying custom Astro environment, organization id, workspace id or organization API token to run these tests when triggered manually from Actions page (if not specified the test setup will pick the default values set as secrets). 6 | 7 | ## Folder Structure 8 | 9 | ``` 10 | e2e-setup/ 11 | ├── astro-project/ 12 | ├── deployment-templates/ 13 | │ ├── deployment-hibernate.yaml 14 | │ └── deployment.yaml 15 | ├── mocks/ 16 | │ ├── dag-deploy-git.sh 17 | │ └── dbt-deploy-git.sh 18 | ├── dbt/ 19 | │ └── dbt_project.yml 20 | ``` 21 | 22 | ### astro-project 23 | 24 | Astro project folder contains a basic airflow project initialized via Astro CLI, which will deployed as part of tests via deploy action 25 | 26 | ### dbt 27 | 28 | The dbt folder contains a basic sample dbt_project.yml file, which is used to define the configuration of a dbt project. This file includes settings such as the project name, version, and other configurations necessary for running dbt commands. 29 | 30 | ### deployment-templates 31 | 32 | Deployment templates contains the basic templates used by e2e tests to create required deployments against which tests would be executed 33 | 34 | - **deployment-hibernate.yaml**: Template for creating a deployment with forever hibernation schedules. 35 | - **deployment.yaml**: Template for creating a standard deployment. 36 | 37 | ### mocks 38 | 39 | mocks folder contain script or logic to mock different part of the logic during e2e tests execution. 40 | As of now it only contain `git.sh` which would replace the git cli so that in deploy action we could mock that only dags file has changed during dag deploy tests. 41 | 42 | - **git.sh**: Script to mock git commands for simulating DAGs-only deploy scenarios. 43 | -------------------------------------------------------------------------------- /e2e-setup/astro-project/.astro/config.yaml: -------------------------------------------------------------------------------- 1 | project: 2 | name: astro-project 3 | -------------------------------------------------------------------------------- /e2e-setup/astro-project/.astro/dag_integrity_exceptions.txt: -------------------------------------------------------------------------------- 1 | # Add dag files to exempt from parse test below. ex: dags/ -------------------------------------------------------------------------------- /e2e-setup/astro-project/.astro/test_dag_integrity_default.py: -------------------------------------------------------------------------------- 1 | """Test the validity of all DAGs. **USED BY DEV PARSE COMMAND DO NOT EDIT**""" 2 | 3 | from contextlib import contextmanager 4 | import logging 5 | import os 6 | 7 | import pytest 8 | 9 | from airflow.models import DagBag, Variable, Connection 10 | from airflow.hooks.base import BaseHook 11 | from airflow.utils.db import initdb 12 | 13 | # init airflow database 14 | initdb() 15 | 16 | # The following code patches errors caused by missing OS Variables, Airflow Connections, and Airflow Variables 17 | 18 | 19 | # =========== MONKEYPATCH BaseHook.get_connection() =========== 20 | def basehook_get_connection_monkeypatch(key: str, *args, **kwargs): 21 | print( 22 | f"Attempted to fetch connection during parse returning an empty Connection object for {key}" 23 | ) 24 | return Connection(key) 25 | 26 | 27 | BaseHook.get_connection = basehook_get_connection_monkeypatch 28 | # # =========== /MONKEYPATCH BASEHOOK.GET_CONNECTION() =========== 29 | 30 | 31 | # =========== MONKEYPATCH OS.GETENV() =========== 32 | def os_getenv_monkeypatch(key: str, *args, **kwargs): 33 | default = None 34 | if args: 35 | default = args[0] # os.getenv should get at most 1 arg after the key 36 | if kwargs: 37 | default = kwargs.get( 38 | "default", None 39 | ) # and sometimes kwarg if people are using the sig 40 | 41 | env_value = os.environ.get(key, None) 42 | 43 | if env_value: 44 | return env_value # if the env_value is set, return it 45 | if ( 46 | key == "JENKINS_HOME" and default is None 47 | ): # fix https://github.com/astronomer/astro-cli/issues/601 48 | return None 49 | if default: 50 | return default # otherwise return whatever default has been passed 51 | return f"MOCKED_{key.upper()}_VALUE" # if absolutely nothing has been passed - return the mocked value 52 | 53 | 54 | os.getenv = os_getenv_monkeypatch 55 | # # =========== /MONKEYPATCH OS.GETENV() =========== 56 | 57 | # =========== MONKEYPATCH VARIABLE.GET() =========== 58 | 59 | 60 | class magic_dict(dict): 61 | def __init__(self, *args, **kwargs): 62 | self.update(*args, **kwargs) 63 | 64 | def __getitem__(self, key): 65 | return {}.get(key, "MOCKED_KEY_VALUE") 66 | 67 | 68 | _no_default = object() # allow falsey defaults 69 | 70 | 71 | def variable_get_monkeypatch(key: str, default_var=_no_default, deserialize_json=False): 72 | print( 73 | f"Attempted to get Variable value during parse, returning a mocked value for {key}" 74 | ) 75 | 76 | if default_var is not _no_default: 77 | return default_var 78 | if deserialize_json: 79 | return magic_dict() 80 | return "NON_DEFAULT_MOCKED_VARIABLE_VALUE" 81 | 82 | 83 | Variable.get = variable_get_monkeypatch 84 | # # =========== /MONKEYPATCH VARIABLE.GET() =========== 85 | 86 | 87 | @contextmanager 88 | def suppress_logging(namespace): 89 | """ 90 | Suppress logging within a specific namespace to keep tests "clean" during build 91 | """ 92 | logger = logging.getLogger(namespace) 93 | old_value = logger.disabled 94 | logger.disabled = True 95 | try: 96 | yield 97 | finally: 98 | logger.disabled = old_value 99 | 100 | 101 | def get_import_errors(): 102 | """ 103 | Generate a tuple for import errors in the dag bag, and include DAGs without errors. 104 | """ 105 | with suppress_logging("airflow"): 106 | dag_bag = DagBag(include_examples=False) 107 | 108 | def strip_path_prefix(path): 109 | return os.path.relpath(path, os.environ.get("AIRFLOW_HOME")) 110 | 111 | # Initialize an empty list to store the tuples 112 | result = [] 113 | 114 | # Iterate over the items in import_errors 115 | for k, v in dag_bag.import_errors.items(): 116 | result.append((strip_path_prefix(k), v.strip())) 117 | 118 | # Check if there are DAGs without errors 119 | for file_path in dag_bag.dags: 120 | # Check if the file_path is not in import_errors, meaning no errors 121 | if file_path not in dag_bag.import_errors: 122 | result.append((strip_path_prefix(file_path), "No import errors")) 123 | 124 | return result 125 | 126 | 127 | @pytest.mark.parametrize( 128 | "rel_path, rv", get_import_errors(), ids=[x[0] for x in get_import_errors()] 129 | ) 130 | def test_file_imports(rel_path, rv): 131 | """Test for import errors on a file""" 132 | if os.path.exists(".astro/dag_integrity_exceptions.txt"): 133 | with open(".astro/dag_integrity_exceptions.txt", "r") as f: 134 | exceptions = f.readlines() 135 | print(f"Exceptions: {exceptions}") 136 | if (rv != "No import errors") and rel_path not in exceptions: 137 | # If rv is not "No import errors," consider it a failed test 138 | raise Exception(f"{rel_path} failed to import with message \n {rv}") 139 | else: 140 | # If rv is "No import errors," consider it a passed test 141 | print(f"{rel_path} passed the import test") 142 | -------------------------------------------------------------------------------- /e2e-setup/astro-project/.dockerignore: -------------------------------------------------------------------------------- 1 | astro 2 | .git 3 | .env 4 | airflow_settings.yaml 5 | logs/ 6 | .venv 7 | airflow.db 8 | airflow.cfg 9 | -------------------------------------------------------------------------------- /e2e-setup/astro-project/.gitignore: -------------------------------------------------------------------------------- 1 | .git 2 | .env 3 | .DS_Store 4 | airflow_settings.yaml 5 | __pycache__/ 6 | astro 7 | .venv 8 | airflow-webserver.pid 9 | webserver_config.py 10 | airflow.cfg 11 | airflow.db 12 | -------------------------------------------------------------------------------- /e2e-setup/astro-project/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/astronomer/astro-runtime:12.7.1 2 | -------------------------------------------------------------------------------- /e2e-setup/astro-project/README.md: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | Welcome to Astronomer! This project was generated after you ran 'astro dev init' using the Astronomer CLI. This readme describes the contents of the project, as well as how to run Apache Airflow on your local machine. 5 | 6 | Project Contents 7 | ================ 8 | 9 | Your Astro project contains the following files and folders: 10 | 11 | - dags: This folder contains the Python files for your Airflow DAGs. By default, this directory includes one example DAG: 12 | - `example_astronauts`: This DAG shows a simple ETL pipeline example that queries the list of astronauts currently in space from the Open Notify API and prints a statement for each astronaut. The DAG uses the TaskFlow API to define tasks in Python, and dynamic task mapping to dynamically print a statement for each astronaut. For more on how this DAG works, see our [Getting started tutorial](https://www.astronomer.io/docs/learn/get-started-with-airflow). 13 | - Dockerfile: This file contains a versioned Astro Runtime Docker image that provides a differentiated Airflow experience. If you want to execute other commands or overrides at runtime, specify them here. 14 | - include: This folder contains any additional files that you want to include as part of your project. It is empty by default. 15 | - packages.txt: Install OS-level packages needed for your project by adding them to this file. It is empty by default. 16 | - requirements.txt: Install Python packages needed for your project by adding them to this file. It is empty by default. 17 | - plugins: Add custom or community plugins for your project to this file. It is empty by default. 18 | - airflow_settings.yaml: Use this local-only file to specify Airflow Connections, Variables, and Pools instead of entering them in the Airflow UI as you develop DAGs in this project. 19 | 20 | Deploy Your Project Locally 21 | =========================== 22 | 23 | 1. Start Airflow on your local machine by running 'astro dev start'. 24 | 25 | This command will spin up 4 Docker containers on your machine, each for a different Airflow component: 26 | 27 | - Postgres: Airflow's Metadata Database 28 | - Webserver: The Airflow component responsible for rendering the Airflow UI 29 | - Scheduler: The Airflow component responsible for monitoring and triggering tasks 30 | - Triggerer: The Airflow component responsible for triggering deferred tasks 31 | 32 | 2. Verify that all 4 Docker containers were created by running 'docker ps'. 33 | 34 | Note: Running 'astro dev start' will start your project with the Airflow Webserver exposed at port 8080 and Postgres exposed at port 5432. If you already have either of those ports allocated, you can either [stop your existing Docker containers or change the port](https://www.astronomer.io/docs/astro/cli/troubleshoot-locally#ports-are-not-available-for-my-local-airflow-webserver). 35 | 36 | 3. Access the Airflow UI for your local Airflow project. To do so, go to http://localhost:8080/ and log in with 'admin' for both your Username and Password. 37 | 38 | You should also be able to access your Postgres Database at 'localhost:5432/postgres'. 39 | 40 | Deploy Your Project to Astronomer 41 | ================================= 42 | 43 | If you have an Astronomer account, pushing code to a Deployment on Astronomer is simple. For deploying instructions, refer to Astronomer documentation: https://www.astronomer.io/docs/astro/deploy-code/ 44 | 45 | Contact 46 | ======= 47 | 48 | The Astronomer CLI is maintained with love by the Astronomer team. To report a bug or suggest a change, reach out to our support. 49 | -------------------------------------------------------------------------------- /e2e-setup/astro-project/dags/.airflowignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/astronomer/deploy-action/6ad2491c264941e15014114a0fa42cb011786de0/e2e-setup/astro-project/dags/.airflowignore -------------------------------------------------------------------------------- /e2e-setup/astro-project/dags/exampledag.py: -------------------------------------------------------------------------------- 1 | """ 2 | ## Astronaut ETL example DAG 3 | 4 | This DAG queries the list of astronauts currently in space from the 5 | Open Notify API and prints each astronaut's name and flying craft. 6 | 7 | There are two tasks, one to get the data from the API and save the results, 8 | and another to print the results. Both tasks are written in Python using 9 | Airflow's TaskFlow API, which allows you to easily turn Python functions into 10 | Airflow tasks, and automatically infer dependencies and pass data. 11 | 12 | The second task uses dynamic task mapping to create a copy of the task for 13 | each Astronaut in the list retrieved from the API. This list will change 14 | depending on how many Astronauts are in space, and the DAG will adjust 15 | accordingly each time it runs. 16 | 17 | For more explanation and getting started instructions, see our Write your 18 | first DAG tutorial: https://www.astronomer.io/docs/learn/get-started-with-airflow 19 | 20 | ![Picture of the ISS](https://www.esa.int/var/esa/storage/images/esa_multimedia/images/2010/02/space_station_over_earth/10293696-3-eng-GB/Space_Station_over_Earth_card_full.jpg) 21 | """ 22 | 23 | from airflow import Dataset 24 | from airflow.decorators import dag, task 25 | from pendulum import datetime 26 | import requests 27 | 28 | 29 | # Define the basic parameters of the DAG, like schedule and start_date 30 | @dag( 31 | start_date=datetime(2024, 1, 1), 32 | schedule="@daily", 33 | catchup=False, 34 | doc_md=__doc__, 35 | default_args={"owner": "Astro", "retries": 3}, 36 | tags=["example"], 37 | ) 38 | def example_astronauts(): 39 | # Define tasks 40 | @task( 41 | # Define a dataset outlet for the task. This can be used to schedule downstream DAGs when this task has run. 42 | outlets=[Dataset("current_astronauts")] 43 | ) # Define that this task updates the `current_astronauts` Dataset 44 | def get_astronauts(**context) -> list[dict]: 45 | """ 46 | This task uses the requests library to retrieve a list of Astronauts 47 | currently in space. The results are pushed to XCom with a specific key 48 | so they can be used in a downstream pipeline. The task returns a list 49 | of Astronauts to be used in the next task. 50 | """ 51 | try: 52 | r = requests.get("http://api.open-notify.org/astros.json") 53 | r.raise_for_status() 54 | number_of_people_in_space = r.json()["number"] 55 | list_of_people_in_space = r.json()["people"] 56 | except: 57 | print("API currently not available, using hardcoded data instead.") 58 | number_of_people_in_space = 12 59 | list_of_people_in_space = [ 60 | {"craft": "ISS", "name": "Oleg Kononenko"}, 61 | {"craft": "ISS", "name": "Nikolai Chub"}, 62 | {"craft": "ISS", "name": "Tracy Caldwell Dyson"}, 63 | {"craft": "ISS", "name": "Matthew Dominick"}, 64 | {"craft": "ISS", "name": "Michael Barratt"}, 65 | {"craft": "ISS", "name": "Jeanette Epps"}, 66 | {"craft": "ISS", "name": "Alexander Grebenkin"}, 67 | {"craft": "ISS", "name": "Butch Wilmore"}, 68 | {"craft": "ISS", "name": "Sunita Williams"}, 69 | {"craft": "Tiangong", "name": "Li Guangsu"}, 70 | {"craft": "Tiangong", "name": "Li Cong"}, 71 | {"craft": "Tiangong", "name": "Ye Guangfu"}, 72 | ] 73 | 74 | context["ti"].xcom_push( 75 | key="number_of_people_in_space", value=number_of_people_in_space 76 | ) 77 | return list_of_people_in_space 78 | 79 | @task 80 | def print_astronaut_craft(greeting: str, person_in_space: dict) -> None: 81 | """ 82 | This task creates a print statement with the name of an 83 | Astronaut in space and the craft they are flying on from 84 | the API request results of the previous task, along with a 85 | greeting which is hard-coded in this example. 86 | """ 87 | craft = person_in_space["craft"] 88 | name = person_in_space["name"] 89 | 90 | print(f"{name} is currently in space flying on the {craft}! {greeting}") 91 | 92 | # Use dynamic task mapping to run the print_astronaut_craft task for each 93 | # Astronaut in space 94 | print_astronaut_craft.partial(greeting="Hello! :)").expand( 95 | person_in_space=get_astronauts() # Define dependencies using TaskFlow API syntax 96 | ) 97 | 98 | 99 | # Instantiate the DAG 100 | example_astronauts() 101 | -------------------------------------------------------------------------------- /e2e-setup/astro-project/packages.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/astronomer/deploy-action/6ad2491c264941e15014114a0fa42cb011786de0/e2e-setup/astro-project/packages.txt -------------------------------------------------------------------------------- /e2e-setup/astro-project/requirements.txt: -------------------------------------------------------------------------------- 1 | # Astro Runtime includes the following pre-installed providers packages: https://www.astronomer.io/docs/astro/runtime-image-architecture#provider-packages 2 | 3 | astro-run-dag # This package is needed for the astro run command. It will be removed before a deploy -------------------------------------------------------------------------------- /e2e-setup/astro-project/tests/dags/test_dag_example.py: -------------------------------------------------------------------------------- 1 | """Example DAGs test. This test ensures that all Dags have tags, retries set to two, and no import errors. This is an example pytest and may not be fit the context of your DAGs. Feel free to add and remove tests.""" 2 | 3 | import os 4 | import logging 5 | from contextlib import contextmanager 6 | import pytest 7 | from airflow.models import DagBag 8 | 9 | 10 | @contextmanager 11 | def suppress_logging(namespace): 12 | logger = logging.getLogger(namespace) 13 | old_value = logger.disabled 14 | logger.disabled = True 15 | try: 16 | yield 17 | finally: 18 | logger.disabled = old_value 19 | 20 | 21 | def get_import_errors(): 22 | """ 23 | Generate a tuple for import errors in the dag bag 24 | """ 25 | with suppress_logging("airflow"): 26 | dag_bag = DagBag(include_examples=False) 27 | 28 | def strip_path_prefix(path): 29 | return os.path.relpath(path, os.environ.get("AIRFLOW_HOME")) 30 | 31 | # prepend "(None,None)" to ensure that a test object is always created even if it's a no op. 32 | return [(None, None)] + [ 33 | (strip_path_prefix(k), v.strip()) for k, v in dag_bag.import_errors.items() 34 | ] 35 | 36 | 37 | def get_dags(): 38 | """ 39 | Generate a tuple of dag_id, in the DagBag 40 | """ 41 | with suppress_logging("airflow"): 42 | dag_bag = DagBag(include_examples=False) 43 | 44 | def strip_path_prefix(path): 45 | return os.path.relpath(path, os.environ.get("AIRFLOW_HOME")) 46 | 47 | return [(k, v, strip_path_prefix(v.fileloc)) for k, v in dag_bag.dags.items()] 48 | 49 | 50 | @pytest.mark.parametrize( 51 | "rel_path,rv", get_import_errors(), ids=[x[0] for x in get_import_errors()] 52 | ) 53 | def test_file_imports(rel_path, rv): 54 | """Test for import errors on a file""" 55 | if rel_path and rv: 56 | raise Exception(f"{rel_path} failed to import with message \n {rv}") 57 | 58 | 59 | APPROVED_TAGS = {} 60 | 61 | 62 | @pytest.mark.parametrize( 63 | "dag_id,dag,fileloc", get_dags(), ids=[x[2] for x in get_dags()] 64 | ) 65 | def test_dag_tags(dag_id, dag, fileloc): 66 | """ 67 | test if a DAG is tagged and if those TAGs are in the approved list 68 | """ 69 | assert dag.tags, f"{dag_id} in {fileloc} has no tags" 70 | if APPROVED_TAGS: 71 | assert not set(dag.tags) - APPROVED_TAGS 72 | 73 | 74 | @pytest.mark.parametrize( 75 | "dag_id,dag, fileloc", get_dags(), ids=[x[2] for x in get_dags()] 76 | ) 77 | def test_dag_retries(dag_id, dag, fileloc): 78 | """ 79 | test if a DAG has retries set 80 | """ 81 | assert ( 82 | dag.default_args.get("retries", None) >= 2 83 | ), f"{dag_id} in {fileloc} must have task retries >= 2." 84 | -------------------------------------------------------------------------------- /e2e-setup/dbt/dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: "jaffle_shop" 2 | 3 | config-version: 2 4 | version: "0.1" 5 | 6 | profile: "jaffle_shop" 7 | 8 | model-paths: ["models"] 9 | seed-paths: ["seeds"] 10 | test-paths: ["tests"] 11 | analysis-paths: ["analysis"] 12 | macro-paths: ["macros"] 13 | 14 | target-path: "target" 15 | clean-targets: 16 | - "target" 17 | - "dbt_modules" 18 | - "logs" 19 | 20 | require-dbt-version: [">=1.0.0", "<2.0.0"] 21 | 22 | models: 23 | jaffle_shop: 24 | materialized: table 25 | staging: 26 | materialized: view 27 | -------------------------------------------------------------------------------- /e2e-setup/deployment-templates/deployment-hibernate.yaml: -------------------------------------------------------------------------------- 1 | deployment: 2 | configuration: 3 | name: deploy-action-hibernate-e2e 4 | description: "" 5 | runtime_version: 12.7.1 6 | dag_deploy_enabled: true 7 | ci_cd_enforcement: false 8 | scheduler_size: SMALL 9 | is_high_availability: false 10 | is_development_mode: true 11 | executor: CELERY 12 | scheduler_count: 1 13 | workspace_name: Deploy Action E2E 14 | deployment_type: STANDARD 15 | cloud_provider: AWS 16 | region: us-west-2 17 | default_task_pod_cpu: "0.25" 18 | default_task_pod_memory: 0.5Gi 19 | resource_quota_cpu: "10" 20 | resource_quota_memory: 20Gi 21 | workload_identity: "" 22 | worker_queues: 23 | - name: default 24 | max_worker_count: 10 25 | min_worker_count: 0 26 | worker_concurrency: 5 27 | worker_type: A5 28 | hibernation_schedules: 29 | - hibernate_at: 0 * * * * 30 | wake_at: 1 * * * * 31 | enabled: true 32 | - hibernate_at: 1 * * * * 33 | wake_at: 0 * * * * 34 | enabled: true 35 | -------------------------------------------------------------------------------- /e2e-setup/deployment-templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | deployment: 2 | configuration: 3 | name: deploy-action-non-dev-e2e 4 | description: "" 5 | runtime_version: 12.7.1 6 | dag_deploy_enabled: true 7 | ci_cd_enforcement: false 8 | scheduler_size: SMALL 9 | is_high_availability: false 10 | is_development_mode: false 11 | executor: CELERY 12 | scheduler_count: 1 13 | workspace_name: Deploy Action E2E 14 | deployment_type: STANDARD 15 | cloud_provider: AWS 16 | region: us-west-2 17 | default_task_pod_cpu: "0.25" 18 | default_task_pod_memory: 0.5Gi 19 | resource_quota_cpu: "10" 20 | resource_quota_memory: 20Gi 21 | workload_identity: "" 22 | worker_queues: 23 | - name: default 24 | max_worker_count: 10 25 | min_worker_count: 0 26 | worker_concurrency: 5 27 | worker_type: A5 28 | -------------------------------------------------------------------------------- /e2e-setup/mocks/astro-deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # pre-req for this mock script would be to have the actual astro cli installed at /usr/local/bin/astro-original 4 | 5 | if [ "$1" = "deploy" ]; then 6 | # Change directory to 'e2e-setup/astro-project' and then call original `astro deploy` 7 | # so that we could simulate the default behavior without needing to have the astro project in base folder 8 | echo "cd into astro project" && cd e2e-setup/astro-project && /usr/local/bin/astro-original "$@" 9 | else 10 | # If it's not a `deploy` command, run the original `astro` 11 | /usr/local/bin/astro-original "$@" 12 | fi 13 | -------------------------------------------------------------------------------- /e2e-setup/mocks/dag-deploy-git.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # hack to mock git commands as part of action.yaml so that we could simulate dags only deploy scenario without making any additional commits 4 | 5 | # Check if the script was invoked with "git diff" 6 | if [[ "$1" == "diff" ]]; then 7 | echo "e2e-setup/astro-project/dags/exampledag.py" 8 | elif [[ "$1" == "fetch" ]]; then 9 | echo "Handling git fetch, doing nothing" 10 | elif [[ "$1" == "cat-file" ]]; then 11 | echo "Handling git cat-file, doing nothing" 12 | else 13 | echo "Error: git mock script isn't configured to handle $1" >&2 14 | exit 1 15 | fi 16 | -------------------------------------------------------------------------------- /e2e-setup/mocks/dbt-deploy-git.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # hack to mock git commands as part of action.yaml so that we could simulate dbt deploy scenario without making any additional commits 4 | 5 | # Check if the script was invoked with "git diff" 6 | if [[ "$1" == "diff" ]]; then 7 | echo "e2e-setup/dbt/dbt_project.yml" 8 | elif [[ "$1" == "fetch" ]]; then 9 | echo "Handling git fetch, doing nothing" 10 | else 11 | echo "Error: git mock script isn't configured to handle $1" >&2 12 | exit 1 13 | fi 14 | -------------------------------------------------------------------------------- /e2e-setup/mocks/image-deploy-git.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # hack to mock git commands as part of action.yaml so that we could simulate image deploy scenario without making any additional commits 4 | 5 | # Check if the script was invoked with "git diff" 6 | if [[ "$1" == "diff" ]]; then 7 | echo "e2e-setup/astro-project/Dockerfile" 8 | elif [[ "$1" == "fetch" ]]; then 9 | echo "Handling git fetch, doing nothing" 10 | elif [[ "$1" == "cat-file" ]]; then 11 | echo "Handling git cat-file, doing nothing" 12 | else 13 | echo "Error: git mock script isn't configured to handle $1" >&2 14 | exit 1 15 | fi 16 | -------------------------------------------------------------------------------- /e2e-setup/mocks/no-deploy-git.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # hack to mock git commands as part of action.yaml so that we could simulate no deploy scenario without making any additional commits 4 | 5 | # Check if the script was invoked with "git diff" 6 | if [[ "$1" == "diff" ]]; then 7 | echo "README.md" 8 | elif [[ "$1" == "fetch" ]]; then 9 | echo "Handling git fetch, doing nothing" 10 | elif [[ "$1" == "cat-file" ]]; then 11 | echo "Handling git cat-file, doing nothing" 12 | else 13 | echo "Error: git mock script isn't configured to handle $1" >&2 14 | exit 1 15 | fi 16 | --------------------------------------------------------------------------------