├── .dockerignore ├── .github └── workflows │ ├── codebuild-runner-dev.yml │ ├── codebuild-runner-release.yml │ ├── codeql-analysis.yml │ ├── python-test.yml │ └── release.yml ├── .gitignore ├── LICENSE.txt ├── README.md ├── __init__.py ├── dg.spec ├── diggercli ├── __init__.py ├── _version.py ├── api.py ├── auth.py ├── constants.py ├── deploy.py ├── dg.py ├── diggerconfig.py ├── env │ ├── .env │ ├── .env.example │ ├── __init__.py │ └── pyinstaller │ │ └── .env ├── exceptions.py ├── fileio.py ├── projects.py ├── server.py ├── templates │ ├── __init__.py │ └── environments │ │ └── local-docker │ │ ├── .postgres.env │ │ ├── __init__.py │ │ └── docker-compose.yml ├── transformers.py ├── utils │ ├── __init__.py │ ├── misc.py │ └── pprint.py └── validators.py ├── docker ├── Dockerfile-release.debian ├── Dockerfile.ecr.publish └── scripts │ └── release.sh ├── requirements.txt ├── scripts └── buildformac.sh ├── setup.py └── tst ├── __init__.py ├── test_cli.py ├── test_config.py ├── test_restrieve_aws_credentials.py └── test_validators.py /.dockerignore: -------------------------------------------------------------------------------- 1 | dist 2 | -------------------------------------------------------------------------------- /.github/workflows/codebuild-runner-dev.yml: -------------------------------------------------------------------------------- 1 | name: Codebuild release ecr for dev 2 | 3 | on: 4 | push: 5 | branches: [dev**, feat/assume-role] 6 | 7 | jobs: 8 | 9 | build-and-release: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | with: 15 | repository: 'diggerhq/codebuild-runner' 16 | token: ${{ secrets.TOKEN_FOR_REPO_CLONE }} 17 | 18 | - name: Configure Staging AWS credentials 19 | uses: aws-actions/configure-aws-credentials@v1 20 | with: 21 | aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} 22 | aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} 23 | aws-region: us-east-1 24 | 25 | - name: Login to STAGING Amazon ECR 26 | id: login-ecr 27 | uses: aws-actions/amazon-ecr-login@v1 28 | 29 | - name: build and push to ecr 30 | run: | 31 | export TAG=$(git rev-parse --short "$GITHUB_SHA") 32 | docker build --build-arg TAG=$TAG -t 682903345738.dkr.ecr.us-east-1.amazonaws.com/codebuild-runner:dev -t 682903345738.dkr.ecr.us-east-1.amazonaws.com/codebuild-runner:$TAG . 33 | docker push "682903345738.dkr.ecr.us-east-1.amazonaws.com/codebuild-runner:$TAG" 34 | docker push "682903345738.dkr.ecr.us-east-1.amazonaws.com/codebuild-runner:dev" 35 | -------------------------------------------------------------------------------- /.github/workflows/codebuild-runner-release.yml: -------------------------------------------------------------------------------- 1 | name: Codebuild release ecr 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 7 | 8 | jobs: 9 | 10 | get-tag-version: 11 | name: Get tag version 12 | outputs: 13 | tag_version: ${{ steps.get_version.outputs.VERSION }} 14 | 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Get the tag version 18 | id: get_version 19 | run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | cut -d / -f 3) 20 | 21 | build-and-release: 22 | needs: get-tag-version 23 | runs-on: ubuntu-latest 24 | 25 | steps: 26 | - uses: actions/checkout@v2 27 | with: 28 | repository: 'diggerhq/codebuild-runner' 29 | token: ${{ secrets.TOKEN_FOR_REPO_CLONE }} 30 | 31 | - name: initialize awscli 32 | uses: aws-actions/configure-aws-credentials@v1 33 | with: 34 | aws-access-key-id: ${{ secrets.AWS_KEY_ID }} 35 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 36 | aws-region: us-east-2 37 | 38 | - name: build and push to ecr 39 | run: | 40 | export TAG=${{needs.get-tag-version.outputs.tag_version}} 41 | export AWS_ACCESS_KEY_ID=${{ secrets.AWS_KEY_ID }} 42 | export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} 43 | aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 739940681129.dkr.ecr.us-east-2.amazonaws.com/codebuild-runner 44 | docker build -t codebuild-runner . 45 | docker tag "codebuild-runner:latest" "739940681129.dkr.ecr.us-east-2.amazonaws.com/codebuild-runner:$TAG" 46 | docker push "739940681129.dkr.ecr.us-east-2.amazonaws.com/codebuild-runner:$TAG" 47 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | 2 | name: "CodeQL" 3 | 4 | on: 5 | push: 6 | branches: [ master ] 7 | pull_request: 8 | branches: [ master ] 9 | schedule: 10 | - cron: '21 11 * * 5' 11 | 12 | jobs: 13 | analyze: 14 | name: Analyze 15 | runs-on: ubuntu-latest 16 | permissions: 17 | actions: read 18 | contents: read 19 | security-events: write 20 | 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | language: [ 'python' ] 25 | 26 | steps: 27 | - name: Checkout repository 28 | uses: actions/checkout@v2 29 | 30 | # Initializes the CodeQL tools for scanning. 31 | - name: Initialize CodeQL 32 | uses: github/codeql-action/init@v1 33 | with: 34 | languages: ${{ matrix.language }} 35 | 36 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 37 | # If this step fails, then you should remove it and run the build manually (see below) 38 | - name: Autobuild 39 | uses: github/codeql-action/autobuild@v1 40 | 41 | # ℹ️ Command-line programs to run using the OS shell. 42 | # 📚 https://git.io/JvXDl 43 | 44 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 45 | # and modify them (or add more) to build your code if your project 46 | # uses a compiled language 47 | 48 | #- run: | 49 | # make bootstrap 50 | # make release 51 | 52 | - name: Perform CodeQL Analysis 53 | uses: github/codeql-action/analyze@v1 54 | -------------------------------------------------------------------------------- /.github/workflows/python-test.yml: -------------------------------------------------------------------------------- 1 | name: Unit Tests 2 | on: [push] 3 | jobs: 4 | test: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - name: Prepare repo 8 | uses: actions/checkout@master 9 | - uses: actions/setup-python@v2 10 | with: 11 | python-version: '3.8' 12 | - name: Test 13 | uses: onichandame/python-test-action@master 14 | with: 15 | deps_list: 'requirements.txt' -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Package and release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 7 | 8 | jobs: 9 | 10 | create-release: 11 | name: Create Release 12 | outputs: 13 | upload_url: ${{ steps.create_release.outputs.upload_url }} 14 | tag_version: ${{ steps.get_version.outputs.VERSION }} 15 | 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Get the tag version 19 | id: get_version 20 | run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | cut -d / -f 3) 21 | 22 | - name: Checkout code 23 | uses: actions/checkout@v2 24 | - name: Create Release 25 | id: create_release 26 | uses: actions/create-release@v1 27 | env: 28 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token 29 | with: 30 | tag_name: ${{ github.ref }} 31 | release_name: Release ${{ github.ref }} 32 | body: | 33 | Changes in this Release 34 | - First Change 35 | - Second Change 36 | draft: true 37 | prerelease: true 38 | 39 | build-and-release-linux: 40 | needs: create-release 41 | runs-on: ubuntu-latest 42 | 43 | steps: 44 | - uses: actions/checkout@v2 45 | 46 | - name: Package Application 47 | uses: diggerhq/pyinstaller-action-linux@main 48 | with: 49 | path: ./ 50 | 51 | - name: Build project # This would actually build your project, using zip for an example artifact 52 | run: | 53 | cd dist/dg-linux 54 | zip -r ../../dg-linux-${{needs.create-release.outputs.tag_version}}.zip dg 55 | cd ../../ 56 | 57 | - name: Upload Release Asset 58 | id: upload-release-asset 59 | uses: actions/upload-release-asset@v1 60 | env: 61 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 62 | with: 63 | upload_url: ${{needs.create-release.outputs.upload_url}} 64 | asset_path: dg-linux-${{needs.create-release.outputs.tag_version}}.zip 65 | asset_name: dg-linux-${{needs.create-release.outputs.tag_version}}.zip 66 | asset_content_type: application/zip 67 | 68 | - name: Upload to S3 69 | uses: aws-actions/configure-aws-credentials@v1 70 | with: 71 | aws-access-key-id: ${{ secrets.AWS_KEY_ID }} 72 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 73 | aws-region: us-east-1 74 | - name: Deploy to S3 75 | run: > 76 | aws s3 cp --acl public-read 77 | dg-linux-${{needs.create-release.outputs.tag_version}}.zip 78 | s3://digger-releases/linux/dg-linux-${{needs.create-release.outputs.tag_version}}.zip 79 | working-directory: . 80 | 81 | 82 | build-and-release-linuxmacos: 83 | needs: create-release 84 | runs-on: macos-latest 85 | 86 | steps: 87 | - uses: actions/checkout@v1 88 | 89 | - name: perform pyinstaller build 90 | run: ./scripts/buildformac.sh 91 | 92 | - name: Build project # This would actually build your project, using zip for an example artifact 93 | run: | 94 | cd dist/dg-mac 95 | zip -r ../../dg-darwin-${{needs.create-release.outputs.tag_version}}.zip dg 96 | cd ../../ 97 | 98 | - name: Upload Release Asset 99 | id: upload-release-asset 100 | uses: actions/upload-release-asset@v1 101 | env: 102 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 103 | with: 104 | upload_url: ${{needs.create-release.outputs.upload_url}} 105 | asset_path: dg-darwin-${{needs.create-release.outputs.tag_version}}.zip 106 | asset_name: dg-darwin-${{needs.create-release.outputs.tag_version}}.zip 107 | asset_content_type: application/zip 108 | 109 | - name: Upload to S3 110 | uses: aws-actions/configure-aws-credentials@v1 111 | with: 112 | aws-access-key-id: ${{ secrets.AWS_KEY_ID }} 113 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 114 | aws-region: eu-west-1 115 | - name: Deploy to S3 116 | run: > 117 | aws s3 cp --acl public-read 118 | dg-darwin-${{needs.create-release.outputs.tag_version}}.zip 119 | s3://digger-releases/darwin/dg-darwin-${{needs.create-release.outputs.tag_version}}.zip 120 | working-directory: . 121 | 122 | update-latest-stable-version: 123 | needs: 124 | - create-release 125 | - build-and-release-linux 126 | - build-and-release-linuxmacos 127 | 128 | runs-on: ubuntu-latest 129 | 130 | steps: 131 | - uses: actions/checkout@v1 132 | 133 | - name: initialize s3 134 | uses: aws-actions/configure-aws-credentials@v1 135 | with: 136 | aws-access-key-id: ${{ secrets.AWS_KEY_ID }} 137 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 138 | aws-region: eu-west-1 139 | 140 | - name: upload version 141 | run: > 142 | echo ${{needs.create-release.outputs.tag_version}} >> STABLE-VERSION && 143 | aws s3 cp --acl public-read 144 | STABLE-VERSION 145 | s3://digger-releases/STABLE-VERSION 146 | working-directory: . 147 | 148 | push-to-ecr: 149 | needs: 150 | - create-release 151 | runs-on: ubuntu-latest 152 | 153 | steps: 154 | - uses: actions/checkout@v1 155 | 156 | - name: initialize awscli 157 | uses: aws-actions/configure-aws-credentials@v1 158 | with: 159 | aws-access-key-id: ${{ secrets.AWS_KEY_ID }} 160 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 161 | aws-region: eu-west-1 162 | 163 | - name: build and push to ecr 164 | run: | 165 | export TAG=${{needs.create-release.outputs.tag_version}} 166 | export AWS_ACCESS_KEY_ID=${{ secrets.AWS_KEY_ID }} 167 | export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} 168 | aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/g1x6q1x1 169 | docker build --build-arg TAG=$TAG -t dg -f docker/Dockerfile.ecr.publish . 170 | docker tag "dg:latest" "public.ecr.aws/g1x6q1x1/dg:$TAG" 171 | docker tag "dg:latest" "public.ecr.aws/g1x6q1x1/dg:latest" 172 | docker push "public.ecr.aws/g1x6q1x1/dg:latest" 173 | docker push "public.ecr.aws/g1x6q1x1/dg:$TAG" 174 | 175 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | __pycache__/ 3 | .python-version 4 | env/.env 5 | dist/ 6 | build/ 7 | .idea/ 8 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Installation (development) 4 | - add an entry in your /etc/hosts to point digger.local -> 127.0.0.1 5 | - create virtualenv with python version > 3.7 6 | - `cp env/.env.example env/.env` 7 | - update endpoint in env/.env file to `https://digger.local:8000` 8 | - pip install -r requirements.txt 9 | - pip install --editable . 10 | - now test with `dg` -- you should see a help screen 11 | - start from a blank folder 12 | 13 | 14 | ## Releasing 15 | 16 | ### On host: 17 | - create a virtualenv for packaging 18 | - pip install -r requirements.txt 19 | - pip insall pyinstaller 20 | - ` pyinstaller dg.spec` 21 | - The released binary will be in dist/ folder 22 | 23 | ### Using docker: 24 | - docker build -t dg-release-debian -f docker/Dockerfile-release.debian . 25 | - docker run -it -v $PWD/dist:/dist dg-release-debian 26 | - The resulting binary will be in the dist/ folder, you can modify this by changing the first argument to `-v` 27 | 28 | ### With github actions: 29 | Any tag which starts with vxxx will be built for linux and malk 30 | A corresponding release will be created. In the releases tab https://github.com/diggerhq/cli/releases 31 | You can download the release and upload to a pulic s3 bucket (we deploy to digger-releases/linux and digger-releases/darwin). 32 | Update [homebrew](https://github.com/diggerhq/homebrew-tap/blob/master/Formula/dg.rb) and [notion page](https://www.notion.so/Quick-Start-deploy-a-service-d55adaf6bcb84399a3ab0633b19a2a45) with latest links 33 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diggerhq/cli/dd2719bf8aaee04c431671cc92a416efc32915cd/__init__.py -------------------------------------------------------------------------------- /dg.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python ; coding: utf-8 -*- 2 | 3 | block_cipher = None 4 | 5 | 6 | a = Analysis(['diggercli/dg.py'], 7 | pathex=['/Users/mohamedsayed/dev/digger/cli'], 8 | binaries=[], 9 | datas=[ 10 | ('diggercli/env/pyinstaller', 'env' ), 11 | ('diggercli/templates', 'templates' ) 12 | ], 13 | hiddenimports=[], 14 | hookspath=[], 15 | runtime_hooks=[], 16 | excludes=[], 17 | win_no_prefer_redirects=False, 18 | win_private_assemblies=False, 19 | cipher=block_cipher, 20 | noarchive=False) 21 | pyz = PYZ(a.pure, a.zipped_data, 22 | cipher=block_cipher) 23 | exe = EXE(pyz, 24 | a.scripts, 25 | [], 26 | exclude_binaries=True, 27 | name='dg', 28 | debug=False, 29 | bootloader_ignore_signals=False, 30 | strip=False, 31 | upx=True, 32 | console=True ) 33 | coll = COLLECT(exe, 34 | a.binaries, 35 | a.zipfiles, 36 | a.datas, 37 | strip=False, 38 | upx=True, 39 | upx_exclude=[], 40 | name='dg') 41 | -------------------------------------------------------------------------------- /diggercli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diggerhq/cli/dd2719bf8aaee04c431671cc92a416efc32915cd/diggercli/__init__.py -------------------------------------------------------------------------------- /diggercli/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.58" 2 | -------------------------------------------------------------------------------- /diggercli/api.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import requests 4 | from requests import Request, Session 5 | from diggercli.constants import BACKEND_ENDPOINT, DIGGERTOKEN_FILE_PATH, DIGGER_ENV_TOKEN_NAME 6 | from diggercli.exceptions import ApiRequestException 7 | from diggercli.utils.pprint import Bcolors 8 | 9 | 10 | def get_github_token(): 11 | env_token = os.environ.get(DIGGER_ENV_TOKEN_NAME, None) 12 | if env_token is not None: 13 | return env_token 14 | 15 | if not os.path.exists(DIGGERTOKEN_FILE_PATH): 16 | return None 17 | f = open(DIGGERTOKEN_FILE_PATH, 'r') 18 | token = f.readline().strip() 19 | return token 20 | 21 | def do_api(method, endpoint, data, stream=False, auth_token=None): 22 | if auth_token is not None: 23 | headers = { 24 | "Authorization": f"Token {auth_token}" 25 | } 26 | else: 27 | headers={} 28 | response = requests.request( 29 | method=method, 30 | stream=stream, 31 | url=endpoint, 32 | json=data, 33 | headers=headers 34 | ) 35 | if response.status_code//100 != 2: 36 | Bcolors.fail("Request failed") 37 | raise ApiRequestException(response.content) 38 | return response 39 | 40 | def check_project_name(projectName): 41 | token = get_github_token() 42 | return do_api( 43 | "GET", 44 | f"{BACKEND_ENDPOINT}/api/check_project_name", 45 | {"project_name": projectName}, 46 | auth_token=token 47 | ) 48 | 49 | 50 | def list_services(projectName): 51 | token = get_github_token() 52 | return do_api( 53 | "GET", 54 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/services/", 55 | {}, 56 | auth_token=token 57 | ) 58 | 59 | def get_service_by_name(projectName, serviceName): 60 | response = list_services(projectName) 61 | service_list = json.loads(response.content)["results"] 62 | for service in service_list: 63 | if service["name"] == serviceName: 64 | return service 65 | raise ApiRequestException("Service not found") 66 | 67 | 68 | def get_service(projectName, servicePk): 69 | token = get_github_token() 70 | return do_api( 71 | "GET", 72 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/services/{servicePk}/", 73 | {}, 74 | auth_token=token 75 | ) 76 | 77 | 78 | def create_service(projectName, data): 79 | token = get_github_token() 80 | return do_api( 81 | "POST", 82 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/services/", 83 | data, 84 | auth_token=token 85 | ) 86 | 87 | 88 | def update_service(projectName, serviceName, data): 89 | token = get_github_token() 90 | return do_api( 91 | "PUT", 92 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/services/{serviceName}/", 93 | data, 94 | auth_token=token 95 | ) 96 | 97 | def sync_services(projectName, data): 98 | token = get_github_token() 99 | return do_api( 100 | "POST", 101 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/services/sync/", 102 | data, 103 | auth_token=token 104 | ) 105 | 106 | 107 | def create_project(projectName): 108 | token = get_github_token() 109 | return do_api( 110 | "POST", 111 | f"{BACKEND_ENDPOINT}/api/projects/", 112 | {"name": projectName}, 113 | auth_token=token 114 | ) 115 | 116 | def generate_project(projectName): 117 | token = get_github_token() 118 | return do_api( 119 | "GET", 120 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/diggeryml/", 121 | {}, 122 | auth_token=token 123 | ) 124 | 125 | 126 | def generate_tmp_project(data): 127 | return do_api( 128 | "POST", 129 | f"{BACKEND_ENDPOINT}/api/tmpProjects/", 130 | data 131 | ) 132 | 133 | def get_signed_url_for_code_upload(uuid, data): 134 | token = get_github_token() 135 | return do_api( 136 | "POST", 137 | f"{BACKEND_ENDPOINT}/api/tmpProjects/{uuid}/code_upload_sign/", 138 | data, 139 | auth_token=token 140 | ) 141 | 142 | def get_project_environments(projectName): 143 | token = get_github_token() 144 | return do_api( 145 | "GET", 146 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/", 147 | {}, 148 | auth_token=token 149 | ) 150 | 151 | def get_environment_details(projectName, environmentName): 152 | response = get_project_environments(projectName) 153 | env_list = json.loads(response.content)["results"] 154 | for env in env_list: 155 | if env["name"] == environmentName: 156 | return env 157 | raise ApiRequestException("Environment not found") 158 | 159 | def create_environment(projectName, data): 160 | token = get_github_token() 161 | return do_api( 162 | "POST", 163 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/", 164 | data, 165 | auth_token=token 166 | ) 167 | 168 | def update_environment(projectName, environmentId, data): 169 | token = get_github_token() 170 | return do_api( 171 | "PUT", 172 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/{environmentId}/", 173 | data, 174 | auth_token=token 175 | ) 176 | 177 | def apply_environment(projectName, environmentId): 178 | token = get_github_token() 179 | return do_api( 180 | "POST", 181 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/{environmentId}/apply/", 182 | {}, 183 | auth_token=token 184 | ) 185 | 186 | 187 | def plan_environment(projectName, environmentId): 188 | token = get_github_token() 189 | return do_api( 190 | "GET", 191 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/{environmentId}/plan/", 192 | {}, 193 | auth_token=token 194 | ) 195 | 196 | def estimate_cost(projectName, environmentId): 197 | token = get_github_token() 198 | return do_api( 199 | "POST", 200 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/{environmentId}/estimate_cost/", 201 | {}, 202 | auth_token=token 203 | ) 204 | 205 | 206 | def environment_vars_list(projectName, environmentId): 207 | token = get_github_token() 208 | return do_api( 209 | "GET", 210 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/{environmentId}/configs/", 211 | {}, 212 | auth_token=token 213 | ) 214 | 215 | 216 | def environment_vars_create(projectName, environmentId, name, value, servicePk, overwrite=False): 217 | token = get_github_token() 218 | return do_api( 219 | "POST", 220 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/{environmentId}/configs/", 221 | { 222 | "name": name, 223 | "value": value, 224 | "service": servicePk, 225 | "overwrite": overwrite, 226 | }, 227 | auth_token=token 228 | ) 229 | 230 | 231 | def destroy_environment(projectName, environmentId, data): 232 | token = get_github_token() 233 | return do_api( 234 | "POST", 235 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/{environmentId}/destroy/", 236 | data, 237 | auth_token=token 238 | ) 239 | 240 | def create_infra(data): 241 | token = get_github_token() 242 | return do_api( 243 | "POST", 244 | f"{BACKEND_ENDPOINT}/api/create", 245 | data, 246 | auth_token=token 247 | ) 248 | 249 | def create_infra_quick(data): 250 | token = get_github_token() 251 | return do_api( 252 | "POST", 253 | f"{BACKEND_ENDPOINT}/api/create_quick", 254 | data, 255 | auth_token=token 256 | ) 257 | 258 | def destroy_infra(data): 259 | token = get_github_token() 260 | return do_api( 261 | "POST", 262 | f"{BACKEND_ENDPOINT}/api/destroy", 263 | data, 264 | auth_token=token 265 | ) 266 | 267 | 268 | def deploy_to_infra(data): 269 | token = get_github_token() 270 | return do_api( 271 | "POST", 272 | f"{BACKEND_ENDPOINT}/api/deploy", 273 | data, 274 | auth_token=token 275 | ) 276 | 277 | def get_job_info(job_id): 278 | token = get_github_token() 279 | return do_api( 280 | "GET", 281 | f"{BACKEND_ENDPOINT}/api/jobs/{job_id}/status", 282 | {}, 283 | auth_token=token 284 | ) 285 | 286 | def get_infra_deployment_info(projectName, deploymentId): 287 | token = get_github_token() 288 | return do_api( 289 | "GET", 290 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/deployments/{deploymentId}/", 291 | {}, 292 | auth_token=token 293 | ) 294 | 295 | def get_deployment_logs(projectName, deploymentId, limit=5000, nextToken=None): 296 | token = get_github_token() 297 | return do_api( 298 | "GET", 299 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/deployments/{deploymentId}/logs?limit={limit}" + (f"&nextToken={nextToken}" if nextToken else ""), 300 | {}, 301 | auth_token=token 302 | ) 303 | 304 | def stream_deployment_logs(projectName, deploymentId): 305 | token = get_github_token() 306 | return do_api( 307 | "GET", 308 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/stream_deployment_logs/{deploymentId}/", 309 | {}, 310 | stream=True, 311 | auth_token=token 312 | ) 313 | 314 | 315 | def perform_service_deploy(projectName, environmentId, serviceId): 316 | token = get_github_token() 317 | return do_api( 318 | "POST", 319 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/{environmentId}/services/{serviceId}/deploy", 320 | {}, 321 | auth_token=token 322 | ) 323 | 324 | 325 | def get_infra_destroy_job_info(projectName, deploymentId): 326 | token = get_github_token() 327 | return do_api( 328 | "GET", 329 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/destroy_jobs/{deploymentId}/", 330 | {}, 331 | auth_token=token 332 | ) 333 | 334 | def get_last_infra_deployment_info(projectName, environmetId): 335 | """ 336 | Retrieves the details of the last deployment for this project + env 337 | """ 338 | token = get_github_token() 339 | return do_api( 340 | "GET", 341 | f"{BACKEND_ENDPOINT}/api/projects/{projectName}/environments/{environmetId}/last_deployment/", 342 | {}, 343 | auth_token=token 344 | ) 345 | 346 | def get_logs(projectName): 347 | token = get_github_token() 348 | return do_api( 349 | "POST", 350 | f"{BACKEND_ENDPOINT}/api/logs", 351 | {"project_name": projectName}, 352 | auth_token=token 353 | ) 354 | 355 | def download_terraform_async(projectName, environment, region, target, services): 356 | token = get_github_token() 357 | return do_api( 358 | "POST", 359 | f"{BACKEND_ENDPOINT}/api/download_terraform", 360 | { 361 | "project_name": projectName, 362 | "environment": environment, 363 | "region": region, 364 | "target": target, 365 | "services": json.dumps(services) 366 | }, 367 | auth_token=token 368 | ) 369 | 370 | def terraform_generate_status(terraformJobId): 371 | token = get_github_token() 372 | return do_api( 373 | "GET", 374 | f"{BACKEND_ENDPOINT}/api/terraform_s3_jobs/{terraformJobId}/status", 375 | {}, 376 | auth_token=token 377 | ) 378 | 379 | def cli_report(payload): 380 | token = get_github_token() 381 | return do_api( 382 | "POST", 383 | f"{BACKEND_ENDPOINT}/api/cli_reporting", 384 | payload, 385 | auth_token=token 386 | ) 387 | -------------------------------------------------------------------------------- /diggercli/auth.py: -------------------------------------------------------------------------------- 1 | import os 2 | import webbrowser 3 | import random 4 | import click 5 | import requests 6 | import urllib 7 | from functools import update_wrapper 8 | from diggercli import diggerconfig 9 | from diggercli.utils.pprint import Bcolors 10 | from diggercli.projects import get_temporary_project_id 11 | from diggercli.constants import ( 12 | GITHUB_LOGIN_ENDPOINT, 13 | WEBAPP_ENDPOINT, 14 | DIGGERTOKEN_FILE_PATH, 15 | DIGGER_ENV_TOKEN_NAME) 16 | from diggercli.server import start_server 17 | from diggercli.fileio import upload_code 18 | 19 | 20 | def save_github_token(token): 21 | f = open(DIGGERTOKEN_FILE_PATH, "w") 22 | f.write(token) 23 | f.close() 24 | 25 | def save_token_and_upload_code(token): 26 | save_github_token(token) 27 | settings = diggerconfig.Generator.load_yaml() 28 | # first_service 29 | key, service = next(iter(settings["services"].items())) 30 | path = os.path.abspath("./") 31 | tmp_project_uuid = get_temporary_project_id() 32 | upload_code(tmp_project_uuid, service["service_name"]) 33 | 34 | def fetch_github_token_with_cli_callback(temporaryProjectId): 35 | """ 36 | Simlar to fetch_github_token but also spins up a local 37 | server to receive the callback from the webui 38 | """ 39 | print('starting server ...') 40 | port = random.randint(8000, 60000) 41 | cli_callback = urllib.parse.quote_plus(f"http://localhost:{port}") 42 | webapp_redirect = urllib.parse.quote_plus(f"{WEBAPP_ENDPOINT}#/init/{temporaryProjectId}") 43 | webbrowser.open(f"{GITHUB_LOGIN_ENDPOINT}?redirect_uri={webapp_redirect}&cli_callback={cli_callback}") 44 | start_server(port, save_token_and_upload_code) 45 | 46 | def fetch_github_token(): 47 | webbrowser.open(GITHUB_LOGIN_ENDPOINT) 48 | token = "" 49 | while len(token) < 1: 50 | Bcolors.warn("Please follow browser and paste token here") 51 | token = input() 52 | save_github_token(token) 53 | Bcolors.okgreen("Authentication successful!") 54 | 55 | def require_auth(func): 56 | @click.pass_context 57 | def wrapper(ctx, *args, **kwargs): 58 | if not os.path.exists(DIGGERTOKEN_FILE_PATH) and \ 59 | not os.environ.get(DIGGER_ENV_TOKEN_NAME, None): 60 | Bcolors.fail("Authentication required, please run `dg auth`") 61 | return 62 | # TODO: figure out why such ctx is not working 63 | # return ctx.invoke(func, ctx.obj, *args, **kwargs) 64 | return func(*args, **kwargs) 65 | return update_wrapper(wrapper, func) 66 | -------------------------------------------------------------------------------- /diggercli/constants.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from pathlib import Path 4 | from environs import Env 5 | 6 | 7 | def get_base_path(): 8 | # for pyinstaller binaries we use sys.MEIPASS otherwise fetch from __file__ 9 | if getattr(sys, 'frozen', False): 10 | return sys._MEIPASS 11 | else: 12 | return os.path.abspath(os.path.dirname(__file__)) 13 | 14 | 15 | DIGGER_SPLASH = """ 16 | 🚀 Digger: Deploy with confidence 🚀 17 | 18 | ______ 19 | (, / ) , 20 | / / _ _ _ __ 21 | _/___ /__(_(_/_(_/__(/_/ (_ 22 | (_/___ / .-/ .-/ 23 | (_/ (_/ 24 | """ 25 | 26 | DIGGER_ENV_TOKEN_NAME = "DIGGER_TOKEN" 27 | BASE_PATH = get_base_path() 28 | HOMEDIR_PATH = str(Path.home()) 29 | DIGGER_CONFIG_FILE = "digger.config.yml" 30 | AWS_HOME_PATH = f"{HOMEDIR_PATH}/.aws" 31 | AWSCREDS_FILE_PATH = f"{AWS_HOME_PATH}/credentials" 32 | DIGGERHOME_PATH = os.path.join(HOMEDIR_PATH, ".digger/") 33 | DIGGERTOKEN_FILE_PATH = f"{DIGGERHOME_PATH}/token" 34 | env = Env() 35 | env.read_env(f"{BASE_PATH}/env/.env", recurse=False) 36 | BACKEND_ENDPOINT = env("BACKEND_ENDPOINT") 37 | GITHUB_LOGIN_ENDPOINT = BACKEND_ENDPOINT + "/login/github/" 38 | WEBAPP_ENDPOINT = env("WEBAPP_ENDPOINT") 39 | PAAS_TARGET = "diggerhq/target-fargate@v1.0.0" 40 | DOCKER_REMOTE_HOST = "ssh://ec2-user@docker-runner.digger.dev" 41 | AWS_REGIONS = [ 42 | "us-east-1", 43 | "us-east-2", 44 | "us-west-1", 45 | "us-west-2", 46 | "af-south-1", 47 | "ap-east-1", 48 | "ap-south-1", 49 | "ap-northeast-3", 50 | "ap-northeast-2", 51 | "ap-northeast-1", 52 | "ap-southeast-1", 53 | "ap-southeast-2", 54 | "ca-central-1", 55 | "cn-north-1", 56 | "cn-northwest-1", 57 | "eu-central-1", 58 | "eu-west-1", 59 | "eu-west-2", 60 | "eu-west-3", 61 | "eu-north-1", 62 | "eu-south-1", 63 | "me-south-1", 64 | "sa-east-1", 65 | "us-gov-east-1", 66 | "us-gov-west-1", 67 | ] 68 | 69 | 70 | class ServiceType: 71 | CONTAINER = "container" 72 | SERVERLESS = "serverless" 73 | WEBAPP = "webapp" 74 | NEXTJS = "nextjs" 75 | -------------------------------------------------------------------------------- /diggercli/deploy.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import io 4 | import time 5 | import zipfile 6 | import boto3 7 | from diggercli.fileio import zipdir 8 | 9 | 10 | def deploy_lambda_function_code( 11 | project_name, 12 | env_name, 13 | service_name, 14 | region, 15 | path, 16 | handler, 17 | aws_key, 18 | aws_secret, 19 | aws_assume_role_arn, 20 | aws_assume_role_external_id, 21 | env_vars={} 22 | ): 23 | buf = io.BytesIO() 24 | ziph = zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) 25 | 26 | cwd = os.getcwd() 27 | os.chdir(path) 28 | try: 29 | zipdir(ziph) 30 | ziph.close() 31 | except Exception as e: 32 | raise e 33 | finally: 34 | os.chdir(cwd) 35 | 36 | 37 | function_name=f"{project_name}-{env_name}-{service_name}" 38 | response = update_handler_and_deploy_lambda(buf.getvalue(), function_name, handler, aws_key, aws_secret, aws_assume_role_arn, aws_assume_role_external_id,region, env_vars=env_vars) 39 | return response 40 | 41 | 42 | def assume_role(role_arn, external_id): 43 | sts_client = boto3.client('sts') 44 | response = sts_client.assume_role( 45 | RoleArn=role_arn, 46 | RoleSessionName="AssumeRoleSession1", 47 | ExternalId=external_id 48 | ) 49 | credentials = response['Credentials'] 50 | return credentials['AccessKeyId'], credentials['SecretAccessKey'], credentials['SessionToken'] 51 | 52 | 53 | def update_handler_and_deploy_lambda(zip_contents, function_name, handler, aws_key, aws_secret, aws_assume_role_arn, aws_assume_role_external_id, region, 54 | env_vars=None): 55 | if env_vars is None: 56 | env_vars = {} 57 | if aws_assume_role_arn: 58 | aws_access_key_id, aws_secret_access_key, aws_session_token = assume_role(aws_assume_role_arn, aws_assume_role_external_id) 59 | 60 | client = boto3.client("lambda", 61 | aws_access_key_id=aws_access_key_id, 62 | aws_secret_access_key=aws_secret_access_key, 63 | aws_session_token=aws_session_token, 64 | region_name=region) 65 | else: 66 | client = boto3.client("lambda", aws_access_key_id=aws_key, aws_secret_access_key=aws_secret, region_name=region) 67 | 68 | client.update_function_configuration( 69 | FunctionName=function_name, 70 | Environment={ 71 | "Variables": env_vars 72 | }, 73 | Handler=handler 74 | ) 75 | # ensure the lambda status is "Successful" before proceeding 76 | cnt = 1 77 | while True: 78 | func_details = client.get_function( 79 | FunctionName=function_name 80 | ) 81 | state = func_details["Configuration"]["LastUpdateStatus"] 82 | if state != "InProgress" or cnt > 20: 83 | break 84 | cnt += 1 85 | time.sleep(5) 86 | response = client.update_function_code( 87 | FunctionName=function_name, 88 | ZipFile=zip_contents, 89 | Publish=True, 90 | DryRun=False, 91 | ) 92 | return response 93 | 94 | 95 | def deploy_nextjs_code( 96 | nextjs_deployment_name, 97 | nextjs_build_dir, 98 | region, 99 | aws_key, 100 | aws_secret, 101 | env_vars={} 102 | ): 103 | config_file = os.path.join(nextjs_build_dir, "config.json") 104 | f = open(config_file, "r") 105 | config = json.loads(f.read()) 106 | 107 | first_key = next(iter(config["lambdas"].keys())) 108 | first_value = next(iter(config["lambdas"].values())) 109 | lambda_key = first_key 110 | lambda_handler = first_value["handler"] 111 | lambda_zip_path = os.path.join(nextjs_build_dir, first_value["filename"]) 112 | lambda_function_name = f"{nextjs_deployment_name}_{lambda_key}" 113 | zip_contents = open(lambda_zip_path, "rb").read() 114 | response = update_handler_and_deploy_lambda(zip_contents, lambda_function_name, lambda_handler, aws_key, aws_secret, region, env_vars=env_vars) 115 | return response 116 | 117 | -------------------------------------------------------------------------------- /diggercli/dg.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | import os 3 | from pprint import pprint 4 | import re 5 | from datetime import datetime 6 | import threading 7 | import shutil 8 | import sys 9 | import time 10 | import json 11 | import configparser 12 | import random 13 | import requests 14 | import click 15 | from pathlib import Path 16 | from collections import OrderedDict 17 | import subprocess 18 | from jinja2 import Template 19 | import yaml 20 | from oyaml import load as yload, dump as ydump 21 | 22 | from diggercli.deploy import deploy_lambda_function_code, deploy_nextjs_code, assume_role 23 | 24 | try: 25 | from yaml import CLoader as Loader, CDumper as Dumper 26 | except ImportError: 27 | from yaml import Loader, Dumper 28 | try: 29 | import importlib.resources as pkg_resources 30 | except ImportError: 31 | # Try backported to PY<37 `importlib_resources`. 32 | import importlib_resources as pkg_resources 33 | 34 | from PyInquirer import prompt as pyprompt, Separator 35 | from diggercli import api 36 | from diggercli.fileio import download_terraform_files 37 | from diggercli.projects import create_temporary_project 38 | from diggercli.auth import ( 39 | fetch_github_token, 40 | require_auth, 41 | fetch_github_token_with_cli_callback 42 | ) 43 | from diggercli.exceptions import CouldNotDetermineDockerLocation, ApiRequestException 44 | from diggercli import diggerconfig 45 | from diggercli.validators import ProjectNameValidator, env_name_validate 46 | from diggercli.transformers import transform_service_name 47 | from diggercli._version import __version__ 48 | from diggercli.constants import ( 49 | PAAS_TARGET, 50 | DIGGER_SPLASH, 51 | DOCKER_REMOTE_HOST, 52 | DIGGERHOME_PATH, 53 | AWS_HOME_PATH, 54 | AWS_REGIONS, 55 | ServiceType, 56 | ) 57 | from diggercli.utils.pprint import Bcolors, Halo, spin, SpinnerSegment 58 | from diggercli.utils.misc import ( 59 | compute_env_vars_with_overrides, 60 | parse_env_config_options, 61 | read_env_config_from_file 62 | ) 63 | 64 | # TODO: use pkg_resources_insead of __file__ since latter will not work for egg 65 | 66 | 67 | PROJECT = {} 68 | 69 | 70 | def digger_yaml(): 71 | return "digger.yml" 72 | 73 | def get_project_settings(): 74 | return yload(open(digger_yaml()), Loader=Loader) 75 | 76 | def update_digger_yaml(d): 77 | global PROJECT 78 | f = open(digger_yaml(), "w") 79 | ydump(d, f) 80 | PROJECT = d 81 | 82 | def find_dockerfile(path): 83 | files = os.listdir(path) 84 | if "Dockerfile" in files: 85 | return os.path.join(path, "Dockerfile") 86 | raise CouldNotDetermineDockerLocation("Could not find dockerfile") 87 | 88 | def dockerfile_manual_entry(service_path): 89 | while True: 90 | print("Please enter path to Dockerfile directly (relative to root service folder)") 91 | path = input() 92 | if os.path.exists(os.path.join(service_path, path)): 93 | return path 94 | else: 95 | print("error, dockerfile not found") 96 | 97 | def prompt_for_aws_keys(currentAwsKey, currentAwsSecret): 98 | if currentAwsKey is None or currentAwsSecret is None: 99 | questions = [ 100 | { 101 | 'type': 'password', 102 | 'name': 'aws_key', 103 | 'message': f'Your AWS Key', 104 | 'validate': lambda x: len(x) > 0 105 | }, 106 | { 107 | 'type': 'password', 108 | 'name': 'aws_secret', 109 | 'message': f'Your AWS Secret', 110 | 'validate': lambda x: len(x) > 0 111 | }, 112 | ] 113 | answers = pyprompt(questions) 114 | else: 115 | maskedAwsKey = currentAwsKey[:4] 116 | maskedAwsSecret = currentAwsSecret[:4] 117 | 118 | questions = [ 119 | { 120 | 'type': 'password', 121 | 'name': 'aws_key', 122 | 'message': f'Your AWS Key ({maskedAwsKey}***)', 123 | }, 124 | { 125 | 'type': 'password', 126 | 'name': 'aws_secret', 127 | 'message': f'Your AWS Secret ({maskedAwsSecret}***)' 128 | }, 129 | ] 130 | 131 | answers = pyprompt(questions) 132 | answers["aws_key"] = currentAwsKey if answers["aws_key"] == "" else answers["aws_key"] 133 | answers["aws_secret"] = currentAwsSecret if answers["aws_secret"] == "" else answers["aws_secret"] 134 | 135 | return answers 136 | 137 | 138 | def get_env_vars(envName, serviceName): 139 | settings = get_project_settings() 140 | envVars = settings["environments"][envName].get("config_vars", {}) 141 | serviceEnvVars = settings["services"][serviceName].get("config_vars", {}).get(envName, {}) 142 | envVars.update(serviceEnvVars) 143 | return envVars 144 | 145 | def retreive_aws_creds(projectName, environment, aws_key=None, aws_secret=None, prompt=True): 146 | diggercredsFile = os.path.join(DIGGERHOME_PATH, "credentials") 147 | profileName = f"{projectName}" 148 | diggerconfig = configparser.ConfigParser() 149 | diggerconfig.read(diggercredsFile) 150 | 151 | if profileName not in diggerconfig: 152 | diggerconfig[profileName] = {} 153 | 154 | if aws_key is not None and aws_secret is not None: 155 | currentAwsKey = aws_key 156 | currentAwsSecret = aws_secret 157 | else: 158 | currentAwsKey = diggerconfig[profileName].get("aws_access_key_id", aws_key) 159 | currentAwsSecret = diggerconfig[profileName].get("aws_secret_access_key", aws_secret) 160 | 161 | if prompt or (currentAwsKey is None or currentAwsSecret is None): 162 | answers = prompt_for_aws_keys(currentAwsKey, currentAwsSecret) 163 | newAwsKey = answers["aws_key"] 164 | newAwsSecret = answers["aws_secret"] 165 | else: 166 | newAwsKey = currentAwsKey 167 | newAwsSecret = currentAwsSecret 168 | 169 | diggerconfig[profileName]["aws_access_key_id"] = newAwsKey 170 | diggerconfig[profileName]["aws_secret_access_key"] = newAwsSecret 171 | 172 | with open(diggercredsFile, 'w') as f: 173 | diggerconfig.write(f) 174 | 175 | return { 176 | "aws_key": newAwsKey, 177 | "aws_secret": newAwsSecret 178 | } 179 | 180 | def generate_docker_compose_file(): 181 | settings = get_project_settings() 182 | services = settings["services"].values() 183 | composeFile = pkg_resources.open_text("diggercli.templates.environments.local-docker", 'docker-compose.yml') 184 | composeContent = composeFile.read() 185 | composeTemplate = Template(composeContent) 186 | 187 | # generate environment files 188 | for service in settings["services"].values(): 189 | for resource in service["resources"].values(): 190 | env_path = "digger-master/local-docker/" 191 | env_file = f"{service['service_name']}_{resource['name']}.env" 192 | # assuming its a database 193 | envFile = pkg_resources.open_text("diggercli.templates.environments.local-docker", f".{resource['engine'].lower()}.env") 194 | envContent = envFile.read() 195 | envTemplate = Template(envContent) 196 | envContentRendered = envTemplate.render({ 197 | "dbhost": resource["name"], 198 | "dbname": resource["name"], 199 | "dbuser": resource["engine"], 200 | "dbpassword": "password", 201 | }) 202 | 203 | envFile = open(f"{env_path}{env_file}", "w") 204 | envFile.write(envContentRendered) 205 | envFile.close() 206 | 207 | resource["env_file"] = env_file 208 | service["env_files"] = service.get("env_files", []) 209 | service["env_files"].append(env_file) 210 | 211 | # including absolute dockerfile path 212 | for service in settings["services"].values(): 213 | service["dockerfile_absolute"] = os.path.abspath(service["dockerfile"]) 214 | service["path_absolute"] = os.path.abspath(service["path"]) 215 | 216 | composeContentRendered = composeTemplate.render({ 217 | "services": services 218 | }) 219 | 220 | composeFile = open("digger-master/local-docker/docker-compose.yml", "w") 221 | composeFile.write(composeContentRendered) 222 | composeFile.close() 223 | 224 | def init_project(project_name): 225 | Path("digger-master").mkdir(parents=True, exist_ok=True) 226 | 227 | settings = OrderedDict() 228 | settings["project"] = { 229 | "name": project_name 230 | } 231 | settings["environments"] = settings.get("environments", {}) 232 | settings["environments"]["local-docker"] = { 233 | "target": "docker" 234 | } 235 | settings["services"] = {} 236 | 237 | return settings 238 | 239 | 240 | def clone_repo(url): 241 | subprocess.Popen(["git", "clone", url]).communicate() 242 | 243 | 244 | def repos(): 245 | return [ 246 | 'todo-backend', 247 | 'todo-frontend', 248 | 'todo-reminder', 249 | ] 250 | 251 | def services(): 252 | return [ 253 | { 254 | "service_name": 'todo-backend', 255 | "service_type": "container", 256 | "service_url": "https://github.com/diggerhq/todo-backend", 257 | }, 258 | { 259 | "service_name": 'todo-frontend', 260 | "service_type": "serverless", 261 | "service_url": "https://github.com/diggerhq/todo-frontend", 262 | }, 263 | # { 264 | # "service_name": 'todo-reminder', 265 | # "service_type": "serverless", 266 | # "service_url": "https://github.com/diggerhq/todo-reminders", 267 | # } 268 | ] 269 | 270 | class DiggerTargets: 271 | FARGATE = "AWS ECS Fargate" 272 | LAMBDA = "AWS lambda (experimental)" 273 | EKS = "(soon!) AWS EKS" 274 | GCR = "(soon!) Google Cloud Run" 275 | OTHER = "other" 276 | 277 | TARGETS = { 278 | FARGATE: "diggerhq/target-fargate@v1.0.4", 279 | LAMBDA: "diggerhq/target-lambda@master", 280 | EKS: "aws_eks", 281 | GCR: "gcp_cloudrun", 282 | OTHER: "other", 283 | } 284 | 285 | def get_service_names(): 286 | return list(map(lambda x: x["service_name"], services())) 287 | 288 | def modes(): 289 | return [ 290 | 'Serverless', 291 | 'Containers', 292 | ] 293 | 294 | def report_async(payload, settings=None, status="start"): 295 | if settings is not None: 296 | payload.update({ 297 | "settings": settings, 298 | }) 299 | payload.update({"status": status}) 300 | x = threading.Thread(target=api.cli_report, args=(payload,)) 301 | x.start() 302 | 303 | def print_version(ctx, param, value): 304 | if value == True: 305 | click.echo(f"dg cli v{__version__}") 306 | ctx.exit() 307 | 308 | class SpecialEpilog(click.Group): 309 | def format_epilog(self, ctx, formatter): 310 | if self.epilog: 311 | formatter.write_paragraph() 312 | for line in self.epilog.split('\n'): 313 | formatter.write_text(line) 314 | 315 | @click.group(cls=SpecialEpilog, epilog=DIGGER_SPLASH) 316 | @click.option('--version', '-v', is_flag=True, is_eager=True, 317 | expose_value=False, callback=print_version) 318 | def cli(): 319 | Path(DIGGERHOME_PATH).mkdir(parents=True, exist_ok=True) 320 | Path(AWS_HOME_PATH).mkdir(parents=True, exist_ok=True) 321 | 322 | @cli.command() 323 | def version(): 324 | """ 325 | Print the current cli version 326 | """ 327 | print("0.1") 328 | 329 | @cli.command() 330 | def auth(): 331 | """ 332 | Authenticate with github 333 | """ 334 | # report_async({"command": f"dg auth"}, status="start") 335 | fetch_github_token() 336 | report_async({"command": f"dg auth"}, status="complete") 337 | 338 | 339 | def validate_project_name(ctx, param, value): 340 | if value is None: 341 | return value 342 | if len(value) > 0: 343 | return value 344 | else: 345 | raise click.BadParameter("Project name required") 346 | 347 | 348 | @cli.group() 349 | @require_auth 350 | def project(): 351 | """ 352 | Configure a new project 353 | """ 354 | 355 | @project.command(name="init") 356 | @click.option("--name", nargs=1, required=False, callback=validate_project_name) 357 | def project_init(name=None): 358 | action = "init" 359 | report_async({"command": f"dg project init"}, status="start") 360 | 361 | if name is None: 362 | defaultProjectName = os.path.basename(os.getcwd()) 363 | questions = [ 364 | { 365 | 'type': 'input', 366 | 'name': 'project_name', 367 | 'message': 'Enter project name', 368 | 'default': defaultProjectName, 369 | 'validate': ProjectNameValidator 370 | }, 371 | ] 372 | 373 | answers = pyprompt(questions) 374 | 375 | project_name = answers["project_name"] 376 | else: 377 | project_name = name 378 | 379 | # This will throw error if project name is invalid (e.g. project exists) 380 | api.check_project_name(project_name) 381 | 382 | spinner = Halo(text='Initializing project: ' + project_name, spinner='dots') 383 | spinner.start() 384 | settings = init_project(project_name) 385 | update_digger_yaml(settings) 386 | spinner.stop() 387 | 388 | 389 | print("project initiated successfully") 390 | report_async({"command": f"dg project init"}, settings=settings, status="copmlete") 391 | 392 | 393 | @cli.group() 394 | @require_auth 395 | def env(): 396 | """ 397 | Configure a new environment 398 | """ 399 | 400 | 401 | @env.command(name="list") 402 | @click.option("--project-name", required=False) 403 | def env_list(project_name=None): 404 | settings = get_project_settings() 405 | report_async({"command": f"dg env list"}, settings=settings, status="start") 406 | 407 | if project_name is None: 408 | if "project" not in settings: 409 | Bcolors.fail("could not load project name from settings") 410 | Bcolors.fail("please pass project via --project-name parameter") 411 | sys.exit(1) 412 | project_name = settings["project"]["name"] 413 | 414 | response = api.get_project_environments(project_name) 415 | environments = json.loads(response.content)["results"] 416 | 417 | for env in environments: 418 | print(f">> {env['name']}") 419 | print(f" -> pk={env['pk']}") 420 | print(f" -> target={env['target']}") 421 | print(f" -> region={env['region']}") 422 | print(f" -> config_options={env['config_options']}") 423 | print(f" -> aws_key={env['aws_key'][:4]}****{env['aws_key'][-4:]}") 424 | report_async({"command": f"dg env list"}, settings=settings, status="complete") 425 | 426 | 427 | @env.command(name="describe") 428 | @click.argument("env_name", nargs=1, required=True) 429 | def env_describe(env_name): 430 | settings = get_project_settings() 431 | report_async({"command": f"dg env details"}, settings=settings, status="start") 432 | 433 | project_name = settings["project"]["name"] 434 | env = api.get_environment_details(project_name, env_name) 435 | envId = env["pk"] 436 | response = api.get_last_infra_deployment_info(project_name, envId) 437 | infraDeploymentDetails = json.loads(response.content) 438 | 439 | print(f">> {env['name']}") 440 | print(f" -> pk={env['pk']}") 441 | print(f" -> target={env['target']}") 442 | print(f" -> region={env['region']}") 443 | print(f" -> config_options={env['config_options']}") 444 | print(f" -> aws_key={env['aws_key'][:4]}****{env['aws_key'][-4:]}") 445 | report_async({"command": f"dg env list"}, settings=settings, status="complete") 446 | pprint(infraDeploymentDetails) 447 | 448 | report_async({"command": f"dg env details"}, settings=settings, status="complete") 449 | import subprocess 450 | 451 | subprocess.STDOUT 452 | 453 | @env.command(name="create") 454 | @click.argument("env_name", nargs=1, required=True) 455 | @click.option("--target", "-t", required=False) 456 | @click.option("--region", "-r", required=False) 457 | @click.option("--config", "-c", multiple=True, required=False) 458 | @click.option("--aws-key", required=False) 459 | @click.option("--aws-secret", required=False) 460 | @click.option('--prompt/--no-prompt', default=True) 461 | def env_create( 462 | env_name, 463 | target=None, 464 | region=None, 465 | aws_key=None, 466 | aws_secret=None, 467 | config=[], 468 | prompt=True 469 | ): 470 | 471 | try: 472 | env_name_validate(env_name) 473 | except ValueError as e: 474 | Bcolors.warn(str(e)) 475 | sys.exit() 476 | 477 | # parsing config options 478 | 479 | cliOptions = parse_env_config_options(config) 480 | try: 481 | configOptions = read_env_config_from_file(env_name, overrideOptions=cliOptions) 482 | except yaml.YAMLError as ex: 483 | print(f"Could not read config file: {exc}") 484 | return 485 | 486 | targets = DiggerTargets.TARGETS 487 | settings = get_project_settings() 488 | report_async({"command": f"dg env create"}, settings=settings, status="start") 489 | project_name = settings["project"]["name"] 490 | 491 | if target is None: 492 | questions = [ 493 | { 494 | 'type': 'list', 495 | 'name': 'target', 496 | 'message': 'Select target', 497 | 'choices': targets.keys() 498 | }, 499 | ] 500 | 501 | answers = pyprompt(questions) 502 | target_key = answers["target"] 503 | target = targets[target_key] 504 | 505 | if target == "other": 506 | 507 | ok = "n" 508 | while (ok.lower() != "y"): 509 | print("Enter target: ", end="") 510 | target = input() 511 | print(f"Confirm Target {target} (Y/N)?", end="") 512 | ok = input() 513 | 514 | elif target_key not in [DiggerTargets.FARGATE, DiggerTargets.LAMBDA]: 515 | Bcolors.fail("This option is currently unsupported! Please try again") 516 | return 517 | else: 518 | # use target from cli arg 519 | target = target 520 | 521 | if region is None: 522 | questions = [ 523 | { 524 | 'type': 'list', 525 | 'name': 'region', 526 | 'message': 'Select region', 527 | 'choices': AWS_REGIONS, 528 | 'default': "us-east-1" 529 | }, 530 | ] 531 | answers = pyprompt(questions) 532 | region = answers["region"] 533 | 534 | if region not in AWS_REGIONS: 535 | Bcolors.fail("This region is not valid! Please try again") 536 | return 537 | 538 | credentials = retreive_aws_creds(project_name, env_name, aws_key=aws_key, aws_secret=aws_secret, prompt=prompt) 539 | aws_key = credentials["aws_key"] 540 | aws_secret = credentials["aws_secret"] 541 | 542 | spinner = Halo(text="Creating environment", spinner="dots") 543 | spinner.start() 544 | 545 | response = api.create_environment(project_name, { 546 | "name": env_name, 547 | "target": target, 548 | "region": region, 549 | "aws_key": aws_key, 550 | "aws_secret": aws_secret, 551 | "config_options": configOptions 552 | }) 553 | spinner.stop() 554 | 555 | Bcolors.okgreen("Environment created successfully") 556 | Bcolors.okgreen(f"Use this command to run it: dg env apply {env_name}") 557 | 558 | @env.command(name="update") 559 | @click.argument("env_name", nargs=1, required=True) 560 | @click.option("--target", "-t", required=False) 561 | @click.option("--config", "-c", multiple=True, required=False) 562 | @click.option("--aws-key", required=False) 563 | @click.option("--aws-secret", required=False) 564 | def env_update(env_name, target=None, config=None, aws_key=None, aws_secret=None): 565 | settings = get_project_settings() 566 | report_async({"command": f"dg env update"}, settings=settings, status="start") 567 | 568 | projectName = settings["project"]["name"] 569 | envDetails = api.get_environment_details(projectName, env_name) 570 | envPk = envDetails["pk"] 571 | 572 | data = {} 573 | if target is not None: 574 | data["target"] = target 575 | if config is not None: 576 | data["config_options"] = parse_env_config_options(config) 577 | cliOptions = parse_env_config_options(config) 578 | try: 579 | configOptions = read_env_config_from_file(env_name, overrideOptions=cliOptions) 580 | except yaml.YAMLError as ex: 581 | print(f"Could not parse config file: {exc}") 582 | return 583 | data["config_options"] = configOptions 584 | data["config_options"] = data["config_options"] 585 | if aws_key is not None: 586 | data["aws_key"] = aws_key 587 | if aws_secret is not None: 588 | data["aws_secret"] = aws_secret 589 | 590 | response = api.update_environment(projectName, envPk, data) 591 | Bcolors.okgreen("environment udpated succesfully") 592 | report_async({"command": f"dg env update"}, settings=settings, status="stop") 593 | 594 | 595 | @env.command(name="apply") 596 | @click.argument("env_name", nargs=1, required=True) 597 | @click.option("--verbose/--no-verbose", default=False) 598 | def env_apply(env_name, verbose): 599 | 600 | settings = get_project_settings() 601 | report_async({"command": f"dg env apply"}, settings=settings, status="start") 602 | projectName = settings["project"]["name"] 603 | envDetails = api.get_environment_details(projectName, env_name) 604 | envPk = envDetails["pk"] 605 | response = api.apply_environment(projectName, envPk) 606 | job = json.loads(response.content) 607 | 608 | # loading until infra status is complete 609 | print("creating infrastructure ...") 610 | spinner = Halo(text="", spinner="dots") 611 | spinner.start() 612 | 613 | if verbose: 614 | with api.stream_deployment_logs(projectName, job['job_id']) as r: 615 | for line in r.iter_lines(): 616 | line = line.decode("utf-8") 617 | print(line) 618 | 619 | while True: 620 | statusResponse = api.get_infra_deployment_info(projectName, job['job_id']) 621 | print(statusResponse.content) 622 | jobStatus = json.loads(statusResponse.content) 623 | 624 | 625 | if jobStatus["status"] == "COMPLETED": 626 | break 627 | elif jobStatus["status"] == "FAILED": 628 | Bcolors.fail("Could not create infrastructure") 629 | print(jobStatus["fail_message"]) 630 | sys.exit(1) 631 | time.sleep(2) 632 | spinner.stop() 633 | 634 | 635 | print("Deployment successful!") 636 | print(f"your deployment details:") 637 | pprint(jobStatus["outputs"]) 638 | 639 | report_async({"command": f"dg env apply"}, settings=settings, status="complete") 640 | 641 | 642 | @env.command(name="plan") 643 | @click.argument("env_name", nargs=1, required=True) 644 | def env_plan(env_name): 645 | 646 | settings = get_project_settings() 647 | report_async({"command": f"dg env plan"}, settings=settings, status="start") 648 | projectName = settings["project"]["name"] 649 | envDetails = api.get_environment_details(projectName, env_name) 650 | envPk = envDetails["pk"] 651 | spinner = Halo(text="Planning environment ...", spinner="dots") 652 | spinner.start() 653 | response = api.plan_environment(projectName, envPk) 654 | spinner.stop() 655 | Bcolors.okgreen("Your environment plan is shown below") 656 | print("--------------------------------") 657 | data = json.loads(response.content) 658 | pprint(data["output"]) 659 | report_async({"command": f"dg env plan"}, settings=settings, status="complete") 660 | 661 | 662 | @env.command(name="cost") 663 | @click.argument("env_name", nargs=1, required=True) 664 | def env_cost(env_name): 665 | 666 | settings = get_project_settings() 667 | report_async({"command": f"dg env cost"}, settings=settings, status="start") 668 | projectName = settings["project"]["name"] 669 | envDetails = api.get_environment_details(projectName, env_name) 670 | envPk = envDetails["pk"] 671 | spinner = Halo(text="Estimating environment costs ...", spinner="dots") 672 | spinner.start() 673 | response = api.estimate_cost(projectName, envPk) 674 | spinner.stop() 675 | Bcolors.okgreen("Your cost estimates are shown below") 676 | print("--------------------------------") 677 | pprint(response.content) 678 | report_async({"command": f"dg env cost"}, settings=settings, status="complete") 679 | 680 | 681 | @env.command(name="sync-tform") 682 | @click.argument("env_name", nargs=1, required=True) 683 | def env_sync_tform(env_name): 684 | settings = get_project_settings() 685 | report_async({"command": f"dg env sync-tform"}, settings=settings, status="start") 686 | project_name = settings["project"]["name"] 687 | services = settings["services"] 688 | env_path = f"digger-master/{env_name}" 689 | tform_path = f"{env_path}/terraform" 690 | target = settings["environments"][env_name]["target"] 691 | region = settings["environments"][env_name]["region"] 692 | Path(env_path).mkdir(parents=True, exist_ok=True) 693 | Path(tform_path).mkdir(parents=True, exist_ok=True) 694 | shutil.rmtree(tform_path) 695 | # tform generation 696 | spinner = Halo(text="Updating terraform ...", spinner="dots") 697 | spinner.start() 698 | download_terraform_files(project_name, env_name, region, target, services, tform_path) 699 | spinner.stop() 700 | Bcolors.okgreen("Terraform updated successfully") 701 | report_async({"command": f"dg env sync-tform"}, settings=settings, status="complete") 702 | 703 | 704 | @env.command(name="vars:list") 705 | @click.argument("env_name", nargs=1, required=True) 706 | def env_vars_list(env_name): 707 | """ 708 | List environment variables for an environment 709 | """ 710 | action = "vars:list" 711 | settings = get_project_settings() 712 | report_async({"command": f"dg env {action}"}, settings=settings, status="start") 713 | project_name = settings["project"]["name"] 714 | envDetails = api.get_environment_details(project_name, env_name) 715 | envId = envDetails["pk"] 716 | envVars = api.environment_vars_list(project_name, envId) 717 | envVars = json.loads(envVars.content)["results"] 718 | report_async({"command": f"dg env {action}"}, settings=settings, status="complete") 719 | pprint(envVars) 720 | 721 | 722 | @env.command(name="vars:create") 723 | @click.argument("env_name", nargs=1, required=True) 724 | @click.option('--file', required=True) 725 | @click.option('--overwrite/--no-overwrite', default=False) 726 | @click.option('--prompt/--no-prompt', default=True) 727 | def env_vars_create(env_name, file, prompt=True, overwrite=False): 728 | """ 729 | Update environment variables for an environment based on .yml file 730 | --overwrite forces overwriting of existing variables 731 | """ 732 | action = "vars:create" 733 | if not os.path.exists(file): 734 | Bcolors.fail("File does not exist") 735 | sys.exit(1) 736 | 737 | settings = get_project_settings() 738 | report_async({"command": f"dg env {action}"}, settings=settings, status="start") 739 | 740 | project_name = settings["project"]["name"] 741 | if prompt and not overwrite: 742 | Bcolors.warn("Note: Environment update will fail if duplicate variables names exist. Proceed? (Y,N)") 743 | Bcolors.okgreen("Hint: If you wish to overwrite existing vars use the --overwrite option along with this command") 744 | 745 | answer = input() 746 | if answer.lower() != "y": 747 | Bcolors.fail("Aborting ...") 748 | sys.exit(1) 749 | 750 | try: 751 | varsToCreate = yload(open(file), Loader=Loader) 752 | except Exception as e: 753 | Bcolors.fail("Error while loading vars file") 754 | print(e) 755 | sys.exit(1) 756 | 757 | envDetails = api.get_environment_details(project_name, env_name) 758 | envId = envDetails["pk"] 759 | 760 | services = api.list_services(project_name) 761 | services = json.loads(services.content)["results"] 762 | servicesDict = {} 763 | for s in services: 764 | servicesDict[s["name"]] = s 765 | 766 | for serviceName, varItems in varsToCreate.items(): 767 | if serviceName == "all": 768 | servicePk = None 769 | else: 770 | if serviceName not in servicesDict.keys(): 771 | Bcolors.fail(f"serviceName not found in backend: {serviceName}") 772 | sys.exit(1) 773 | servicePk = servicesDict[serviceName]["pk"] 774 | 775 | Bcolors.okgreen(f"Creating vars for service: {serviceName}:") 776 | for varName, varValue in varItems.items(): 777 | Bcolors.okgreen(f"> Creating var ({varName}, {varValue}) ...") 778 | response = api.environment_vars_create( 779 | project_name, 780 | envId, 781 | varName, 782 | varValue, 783 | servicePk, 784 | overwrite=overwrite 785 | ) 786 | Bcolors.okgreen(f">> Created!") 787 | 788 | 789 | report_async({"command": f"dg env {action}"}, settings=settings, status="complete") 790 | 791 | 792 | @env.command(name="build") 793 | @click.argument("env_name", nargs=1, required=True) 794 | @click.option('--service', default=None) 795 | @click.option('--remote/--no-remote', default=False) 796 | @click.option('--tag', default="latest") 797 | @click.option('--context', default=None) 798 | def env_build(env_name, service, remote, context=None, tag="latest"): 799 | action = "build" 800 | settings = get_project_settings() 801 | report_async({"command": f"dg env {action}"}, settings=settings, status="start") 802 | 803 | 804 | if service is None: 805 | defaultProjectName = os.path.basename(os.getcwd()) 806 | questions = [ 807 | { 808 | 'type': 'list', 809 | 'name': 'service_name', 810 | 'message': 'Select Service', 811 | 'choices': settings["services"].keys(), 812 | }, 813 | ] 814 | 815 | answers = pyprompt(questions) 816 | 817 | service_key = answers["service_name"] 818 | else: 819 | service_key = service 820 | 821 | project_name = settings["project"]["name"] 822 | service_name = settings["services"][service_key]["service_name"] 823 | service_type = settings["services"][service_key]["service_type"] 824 | webapp_package_manager = settings["services"][service_key]["webapp_package_manager"] 825 | service_runtime = settings["services"][service_key]["lambda_runtime"] 826 | service_path = settings["services"][service_key]["path"] 827 | envDetails = api.get_environment_details(project_name, env_name) 828 | envId = envDetails["pk"] 829 | exposeVarsAtBuild = envDetails["inject_env_variables_at_build_time"] 830 | 831 | if context is None: 832 | context = f"{service_path}/" 833 | 834 | envVars = api.environment_vars_list(project_name, envId) 835 | envVars = json.loads(envVars.content)["results"] 836 | 837 | serviceDetails = api.get_service_by_name(project_name, service_name) 838 | servicePk = serviceDetails["pk"] 839 | 840 | if service_type in [ServiceType.WEBAPP, ServiceType.NEXTJS]: 841 | build_command = settings["services"][service_key]["build_command"] 842 | 843 | envVarsWithOverrides = compute_env_vars_with_overrides(envVars, servicePk) 844 | # expose env variables 845 | for name, value in envVarsWithOverrides.items(): 846 | os.environ[name] = value 847 | 848 | # run it in service context 849 | if webapp_package_manager == "yarn": 850 | subprocess.run(["yarn", "install", "--prefix", context], check=True) 851 | else: 852 | subprocess.run(["npm", "install", "--prefix", context], check=True) 853 | 854 | print(f"build command to execute: {build_command}") 855 | # ensure that && separator works as expected 856 | for cmd in build_command.split("&&"): 857 | current_cmd = cmd.strip().split(" ") 858 | if current_cmd[0] == "npm": 859 | current_cmd = current_cmd + ["--prefix", context] 860 | subprocess.run(current_cmd, check=True) 861 | 862 | subprocess.run(["pwd"], check=True) 863 | subprocess.run(["ls", "-a"], check=True) 864 | 865 | elif service_type == ServiceType.CONTAINER or (service_type == ServiceType.SERVERLESS and service_runtime == "Docker"): 866 | dockerfile = settings["services"][service_key]["dockerfile"] 867 | response = api.get_last_infra_deployment_info(project_name, envId) 868 | infraDeploymentDetails = json.loads(response.content) 869 | docker_registry = infraDeploymentDetails["outputs"]["services"][service_name]["docker_registry"] 870 | 871 | if remote: 872 | os.environ["DOCKER_HOST"] = DOCKER_REMOTE_HOST 873 | 874 | buildArgs = [] 875 | if exposeVarsAtBuild: 876 | envVarsWithOverrides = compute_env_vars_with_overrides(envVars, servicePk) 877 | 878 | for name, value in envVarsWithOverrides.items(): 879 | os.environ[name] = value 880 | buildArgs = buildArgs + ["--build-arg", name] 881 | 882 | docker_build_command = ["docker", "build", "-t", f"{project_name}-{service_name}:{tag}"] + \ 883 | buildArgs + \ 884 | ["-f", f"{dockerfile}", context] 885 | 886 | subprocess.run(docker_build_command, check=True) 887 | subprocess.run(["docker", "tag", f"{project_name}-{service_name}:{tag}", f"{docker_registry}:{tag}"], check=True) 888 | else: 889 | Bcolors.warn(f"This service type does not support build phase: {service_type}, skipping ...") 890 | sys.exit(0) 891 | 892 | report_async({"command": f"dg env {action}"}, settings=settings, status="complete") 893 | 894 | 895 | @env.command(name="push") 896 | @click.argument("env_name", nargs=1, required=True) 897 | @click.option('--service', default=None) 898 | @click.option("--aws-key", required=False) 899 | @click.option("--aws-secret", required=False) 900 | @click.option("--aws-assume-role-arn", required=False) 901 | @click.option("--aws-assume-external-id", required=False) 902 | @click.option('--remote/--no-remote', default=False) 903 | @click.option('--tag', default="latest") 904 | @click.option('--prompt/--no-prompt', default=False) 905 | def env_push(env_name, service, remote, aws_key=None, aws_secret=None, aws_assume_role_arn=None, aws_assume_external_id=None, tag="latest", prompt=False): 906 | action = "push" 907 | settings = get_project_settings() 908 | report_async({"command": f"dg env {action}"}, settings=settings, status="start") 909 | 910 | if service is None: 911 | questions = [ 912 | { 913 | 'type': 'list', 914 | 'name': 'service_name', 915 | 'message': 'Select Service', 916 | 'choices': settings["services"].keys(), 917 | }, 918 | ] 919 | 920 | answers = pyprompt(questions) 921 | 922 | service_key = answers["service_name"] 923 | else: 924 | service_key = service 925 | 926 | project_name = settings["project"]["name"] 927 | service_name = settings["services"][service_key]["service_name"] 928 | service_type = settings["services"][service_key]["service_type"] 929 | service_runtime = settings["services"][service_key]["lambda_runtime"] 930 | 931 | if service_type == ServiceType.CONTAINER or (service_type == ServiceType.SERVERLESS and service_runtime == "Docker"): 932 | envDetails = api.get_environment_details(project_name, env_name) 933 | envId = envDetails["pk"] 934 | response = api.get_last_infra_deployment_info(project_name, envId) 935 | infraDeploymentDetails = json.loads(response.content) 936 | 937 | if remote: 938 | os.environ["DOCKER_HOST"] = DOCKER_REMOTE_HOST 939 | 940 | docker_registry = infraDeploymentDetails["outputs"]["services"][service_name]["docker_registry"] 941 | region = infraDeploymentDetails["region"] 942 | registry_endpoint = docker_registry.split("/")[0] 943 | if aws_assume_role_arn: 944 | (access_key, secret_key, session_token) = assume_role(aws_assume_role_arn, aws_assume_external_id) 945 | os.environ["AWS_ACCESS_KEY_ID"] = access_key 946 | os.environ["AWS_SECRET_ACCESS_KEY"] = secret_key 947 | os.environ["AWS_SESSION_TOKEN"] = session_token 948 | else: 949 | credentials = retreive_aws_creds(project_name, env_name, aws_key=aws_key, aws_secret=aws_secret, prompt=prompt) 950 | os.environ["AWS_ACCESS_KEY_ID"] = credentials["aws_key"] 951 | os.environ["AWS_SECRET_ACCESS_KEY"] = credentials["aws_secret"] 952 | proc = subprocess.run(["aws", "ecr", "get-login-password", "--region", region, ], capture_output=True) 953 | docker_auth = proc.stdout.decode("utf-8") 954 | subprocess.run(["docker", "login", "--username", "AWS", "--password", docker_auth, registry_endpoint], check=True) 955 | subprocess.run(["docker", "push", f"{docker_registry}:{tag}"], check=True) 956 | elif service_type == ServiceType.NEXTJS: 957 | print(f"ServiceType is NextJS, do nothing for now.") 958 | else: 959 | Bcolors.warn(f"This service: {service_type} does not support push command, skipping ...") 960 | sys.exit(0) 961 | 962 | report_async({"command": f"dg env {action}"}, settings=settings, status="complete") 963 | 964 | 965 | @env.command(name="release") 966 | @click.argument("env_name", nargs=1, required=True) 967 | @click.option('--service', default=None) 968 | @click.option('--all-services/--not-all-services', default=False) 969 | @click.option("--aws-key", required=False) 970 | @click.option("--aws-secret", required=False) 971 | @click.option("--aws-assume-role-arn", required=False) 972 | @click.option("--aws-assume-external-id", required=False) 973 | @click.option('--prompt/--no-prompt', default=False) 974 | @click.option('--tag', default="latest") 975 | def env_release(env_name, service, tag="latest", aws_key=None, aws_secret=None, aws_assume_role_arn=None, aws_assume_external_id=None, all_services=False, prompt=False): 976 | 977 | 978 | def perform_release(settings, env_name, service_key): 979 | project_name = settings["project"]["name"] 980 | service_name = settings["services"][service_key]["service_name"] 981 | service_type = settings["services"][service_key]["service_type"] 982 | service_path = settings["services"][service_key]["path"] 983 | service_runtime = settings["services"][service_key]["lambda_runtime"] 984 | envDetails = api.get_environment_details(project_name, env_name) 985 | envId = envDetails["pk"] 986 | region = envDetails["region"] 987 | 988 | response = api.get_last_infra_deployment_info(project_name, envId) 989 | infraDeploymentDetails = json.loads(response.content) 990 | 991 | awsKey, awsSecret = None, None 992 | if aws_assume_role_arn: 993 | (access_key, secret_key, session_token) = assume_role(aws_assume_role_arn, aws_assume_external_id) 994 | os.environ["AWS_ACCESS_KEY_ID"] = access_key 995 | os.environ["AWS_SECRET_ACCESS_KEY"] = secret_key 996 | os.environ["AWS_SESSION_TOKEN"] = session_token 997 | else: 998 | credentials = retreive_aws_creds(project_name, env_name, aws_key=aws_key, aws_secret=aws_secret, 999 | prompt=prompt) 1000 | awsKey = credentials["aws_key"] 1001 | awsSecret = credentials["aws_secret"] 1002 | os.environ["AWS_ACCESS_KEY_ID"] = awsKey 1003 | os.environ["AWS_SECRET_ACCESS_KEY"] = awsSecret 1004 | 1005 | envVars = {} #get_env_vars(env_name, service_key) 1006 | 1007 | spinner = Halo(text=f"deploying {service_name}...", spinner="dots") 1008 | spinner.start() 1009 | if service_type == ServiceType.WEBAPP: 1010 | build_directory = settings["services"][service_key]["build_directory"] 1011 | # TODO: find better way to extract bucket name of webapp 1012 | bucket_name = infraDeploymentDetails["terraform_outputs"][f"{service_name}_bucket_main"]["value"] 1013 | 1014 | subprocess.run(["aws", "s3", "sync", f"{build_directory}", f"s3://{bucket_name}"], check=True) 1015 | 1016 | Bcolors.okgreen("Upload succeeded!") 1017 | elif service_type == ServiceType.CONTAINER or (service_type == ServiceType.SERVERLESS and service_runtime == "Docker"): 1018 | docker_registry = infraDeploymentDetails["outputs"]["services"][service_name]["docker_registry"] 1019 | lb_url = infraDeploymentDetails["outputs"]["services"][service_name]["lb_url"] 1020 | region = infraDeploymentDetails["region"] 1021 | 1022 | response = api.deploy_to_infra({ 1023 | "environment_pk": f"{envId}", 1024 | "cluster_name": f"{project_name}-{env_name}", 1025 | "service_name": f"{service_name}", 1026 | "task_name": f"{project_name}-{env_name}-{service_name}", 1027 | "region": region, 1028 | "image_url": f"{docker_registry}:{tag}", 1029 | "tag": tag, 1030 | "aws_key": awsKey, 1031 | "aws_secret": awsSecret, 1032 | "aws_assume_role_arn": aws_assume_role_arn, 1033 | "aws_assume_external_id": aws_assume_external_id, 1034 | "env_vars": envVars 1035 | }) 1036 | 1037 | output = json.loads(response.content) 1038 | 1039 | print(output["msg"]) 1040 | print(f"your deployment URL: http://{lb_url}") 1041 | elif service_type == ServiceType.SERVERLESS and service_runtime != "Docker": 1042 | # perform deployment for lambda functions that are not using docker runtime 1043 | if service_runtime == "Node.js": 1044 | print("Installing packages ...") 1045 | # we pass the `--only-production` flag to avoid installing dev dependencies 1046 | subprocess.run(["npm", "i", "--only=production", "--prefix", service_path]) 1047 | elif service_runtime == "Python3.9": 1048 | print("Installing packages ...") 1049 | # needs more work .. we need to include python requirements folder into the zip path 1050 | reqs_path = os.path.join(service_path, "requirements.txt") 1051 | deps_path = service_path 1052 | subprocess.run(["pip", "install", "--target", deps_path, "-r", reqs_path]) 1053 | 1054 | serviceDetails = api.get_service_by_name(project_name, service_name) 1055 | servicePk = serviceDetails["pk"] 1056 | envVars = api.environment_vars_list(project_name, envId) 1057 | envVars = json.loads(envVars.content)["results"] 1058 | envVarsWithOverrides = compute_env_vars_with_overrides(envVars, servicePk) 1059 | 1060 | lambda_handler = settings["services"][service_key]["lambda_handler"] 1061 | response = deploy_lambda_function_code( 1062 | project_name, 1063 | env_name, 1064 | service_name, 1065 | region, 1066 | service_path, 1067 | lambda_handler, 1068 | awsKey, 1069 | awsSecret, 1070 | aws_assume_role_arn, 1071 | aws_assume_external_id, 1072 | env_vars=envVarsWithOverrides 1073 | ) 1074 | print(f"lambda deployed successfully {response}") 1075 | elif service_type == ServiceType.NEXTJS: 1076 | nextjs_deployment_name = infraDeploymentDetails["terraform_outputs"]["nextjs_deployment_name"]["value"] 1077 | nextjs_build_dir = settings["services"][service_key]["build_directory"] 1078 | 1079 | response = deploy_nextjs_code( 1080 | nextjs_deployment_name, 1081 | nextjs_build_dir, 1082 | region, 1083 | awsKey, 1084 | awsSecret, 1085 | ) 1086 | print(f"nextjs app deployed successfully {response}") 1087 | 1088 | else: 1089 | Bcolors.warn(f"Service type: {service_type} does not support release command, skipping ...") 1090 | 1091 | spinner.stop() 1092 | 1093 | action = "deploy" 1094 | settings = get_project_settings() 1095 | report_async({"command": f"dg env {action}"}, settings=settings, status="start") 1096 | 1097 | if all_services: 1098 | service_keys = list(settings["services"].keys()) 1099 | elif service is not None: 1100 | service_keys = [service] 1101 | else: 1102 | defaultProjectName = os.path.basename(os.getcwd()) 1103 | questions = [ 1104 | { 1105 | 'type': 'list', 1106 | 'name': 'service_name', 1107 | 'message': 'Select Service', 1108 | 'choices': settings["services"].keys(), 1109 | }, 1110 | ] 1111 | 1112 | answers = pyprompt(questions) 1113 | 1114 | service_keys = [answers["service_name"]] 1115 | 1116 | for service_key in service_keys: 1117 | perform_release(settings, env_name, service_key) 1118 | 1119 | report_async({"command": f"dg env {action}"}, settings=settings, status="complete") 1120 | 1121 | @env.command(name="software_deploy") 1122 | @click.argument("env_name", nargs=1, required=True) 1123 | @click.option('--service', default=None) 1124 | @click.option('--prompt/--no-prompt', default=False) 1125 | def env_service_deploy(env_name, service, prompt=False): 1126 | settings = get_project_settings() 1127 | if service is None: 1128 | questions = [ 1129 | { 1130 | 'type': 'list', 1131 | 'name': 'service_name', 1132 | 'message': 'Select Service', 1133 | 'choices': settings["services"].keys(), 1134 | }, 1135 | ] 1136 | 1137 | answers = pyprompt(questions) 1138 | service_key = answers["service_name"] 1139 | else: 1140 | service_key = service 1141 | 1142 | 1143 | projectName = settings["project"]["name"] 1144 | envDetails = api.get_environment_details(projectName, env_name) 1145 | environmentId = envDetails["pk"] 1146 | serviceDetails = api.get_service_by_name(projectName, service_key) 1147 | serviceId = serviceDetails["pk"] 1148 | 1149 | with SpinnerSegment(f"Triggering software deploy ..."): 1150 | response = api.perform_service_deploy(projectName, environmentId, serviceId) 1151 | data = json.loads(response.content) 1152 | deploymentId = data["job_id"] 1153 | 1154 | # streaming logs until deployment is completed 1155 | nextToken = None 1156 | monitoring_max_retries = 60 1157 | monitoring_current_retry_count = 1 1158 | with SpinnerSegment(f"Streaming logs ..."): 1159 | while True: 1160 | details_response = api.get_infra_deployment_info(projectName, deploymentId) 1161 | details_data = json.loads(details_response.content) 1162 | status = details_data["status"] 1163 | 1164 | logs_response = api.get_deployment_logs(projectName, deploymentId, limit=5000, nextToken=nextToken) 1165 | logs_data = json.loads(logs_response.content) 1166 | for log_record in logs_data["events"]: 1167 | sys.stdout.write(log_record["message"]) 1168 | 1169 | nextToken = logs_data.get("nextToken", None) 1170 | 1171 | if status in ["LIVE", "COMPLETED", "FAILED"] or monitoring_current_retry_count > monitoring_max_retries: 1172 | break 1173 | 1174 | monitoring_current_retry_count += 1 1175 | time.sleep(1) 1176 | 1177 | live_max_retries = 60 1178 | live_current_retry_count = 1 1179 | with SpinnerSegment("waiting for deployment to be live ..."): 1180 | while True: 1181 | Bcolors.warn("... still waiting for deployment to be live ...") 1182 | details_response = api.get_infra_deployment_info(projectName, deploymentId) 1183 | details_data = json.loads(details_response.content) 1184 | status = details_data["status"] 1185 | 1186 | 1187 | if status == "LIVE" or live_current_retry_count > live_max_retries: 1188 | break 1189 | 1190 | live_current_retry_count += 1 1191 | time.sleep(10) 1192 | 1193 | if status == "LIVE": 1194 | Bcolors.okgreen("** SUCCESS! Your service is now live :) **") 1195 | else: 1196 | Bcolors.fail("Deployment status didn't go live in time :( Please check logs in digger dashboard") 1197 | 1198 | 1199 | 1200 | @env.command(name="destroy") 1201 | @click.argument("env_name", nargs=1, required=True) 1202 | @click.option("--project-name", required=False) 1203 | @click.option("--aws-key", required=False) 1204 | @click.option("--aws-secret", required=False) 1205 | @click.option('--prompt/--no-prompt', default=True) 1206 | def env_destroy(env_name, project_name=None, aws_key=None, aws_secret=None, prompt=True): 1207 | 1208 | settings = get_project_settings() 1209 | report_async({"command": f"dg env destroy"}, settings=settings, status="start") 1210 | projectName = settings["project"]["name"] 1211 | envDetails = api.get_environment_details(projectName, env_name) 1212 | envPk = envDetails["pk"] 1213 | response = api.destroy_environment(projectName, envPk, { 1214 | "aws_key": aws_key, 1215 | "aws_secret": aws_secret 1216 | }) 1217 | job = json.loads(response.content) 1218 | 1219 | 1220 | if prompt: 1221 | questions = [ 1222 | { 1223 | 'type': 'input', 1224 | 'name': 'sure', 1225 | 'message': 'Are you sure (Y/N)?' 1226 | }, 1227 | ] 1228 | 1229 | answers = pyprompt(questions) 1230 | if answers["sure"] != "Y": 1231 | Bcolors.fail("aborting") 1232 | sys.exit(1) 1233 | 1234 | # loading until infra status is complete 1235 | spinner = Halo(text="destroying infrastructure ...", spinner="dots") 1236 | spinner.start() 1237 | while True: 1238 | statusResponse = api.get_infra_destroy_job_info(projectName, job['job_id']) 1239 | print(statusResponse.content) 1240 | jobStatus = json.loads(statusResponse.content) 1241 | if jobStatus["status"] == "DESTROYED": 1242 | break 1243 | elif jobStatus["status"] == "FAILED": 1244 | Bcolors.fail("Could not destroy infrastructure") 1245 | print(jobStatus["fail_message"]) 1246 | sys.exit(1) 1247 | time.sleep(2) 1248 | spinner.stop() 1249 | 1250 | 1251 | print(f"Environment destroyed succesfully") 1252 | report_async({"command": f"dg env destroy"}, settings=settings, status="complete") 1253 | 1254 | 1255 | @env.command(name="history") 1256 | def env_history(): 1257 | action = "history" 1258 | print("Not implemented yet") 1259 | 1260 | @env.command(name="up") 1261 | @click.argument("env_name", nargs=1, default="local-docker") 1262 | def env_up(env_name): 1263 | action = "up" 1264 | report_async({"command": f"dg env {action}"}, status="start") 1265 | if env_name == "local-docker": 1266 | subprocess.Popen(["docker-compose", "-f", "digger-master/local-docker/docker-compose.yml", "up"]).communicate() 1267 | else: 1268 | print("Not implemented yet") 1269 | report_async({"command": f"dg env {action}"}, status="complete") 1270 | 1271 | 1272 | def validate_project_name(ctx, param, value): 1273 | if value is None: 1274 | return value 1275 | if len(value) > 0: 1276 | return value 1277 | else: 1278 | raise click.BadParameter("Project name required") 1279 | 1280 | @cli.group() 1281 | @require_auth 1282 | def project(): 1283 | """ 1284 | Configure a new project 1285 | """ 1286 | 1287 | @project.command(name="init") 1288 | @click.option("--name", nargs=1, required=False, callback=validate_project_name) 1289 | def project_init(name=None): 1290 | action = "init" 1291 | report_async({"command": f"dg project init"}, status="start") 1292 | 1293 | update_existing_yaml = False 1294 | if os.path.exists("digger.yml"): 1295 | Bcolors.warn("digger.yml found, would you like to initialize new project (Y/N)? ") 1296 | answer = input() 1297 | if answer.lower() == "n": 1298 | Bcolors.fail("aborting ...") 1299 | sys.exit(1) 1300 | else: 1301 | update_existing_yaml = True 1302 | 1303 | if name is None: 1304 | defaultProjectName = os.path.basename(os.getcwd()) 1305 | questions = [ 1306 | { 1307 | 'type': 'input', 1308 | 'name': 'project_name', 1309 | 'message': 'Enter project name', 1310 | 'default': defaultProjectName, 1311 | 'validate': ProjectNameValidator 1312 | }, 1313 | ] 1314 | 1315 | answers = pyprompt(questions) 1316 | 1317 | project_name = answers["project_name"] 1318 | else: 1319 | project_name = name 1320 | 1321 | # This will throw error if project name is invalid (e.g. project exists) 1322 | api.create_project(project_name) 1323 | 1324 | spinner = Halo(text='Initializing project: ' + project_name, spinner='dots') 1325 | spinner.start() 1326 | if update_existing_yaml: 1327 | settings = get_project_settings() 1328 | settings["project"]["name"] = project_name 1329 | else: 1330 | settings = init_project(project_name) 1331 | update_digger_yaml(settings) 1332 | spinner.stop() 1333 | 1334 | 1335 | print("project initiated successfully") 1336 | report_async({"command": f"dg project init"}, settings=settings, status="copmlete") 1337 | 1338 | 1339 | 1340 | @project.command(name="generate") 1341 | @click.option("--name", nargs=1, required=False, callback=validate_project_name) 1342 | def project_generate_yml(name=None): 1343 | action = "init" 1344 | report_async({"command": f"dg project generate"}, status="start") 1345 | 1346 | update_existing_yaml = False 1347 | if os.path.exists("digger.yml"): 1348 | Bcolors.fail("digger.yml found, please remove before running command") 1349 | sys.exit(1) 1350 | 1351 | if name is None: 1352 | defaultProjectName = os.path.basename(os.getcwd()) 1353 | questions = [ 1354 | { 1355 | 'type': 'input', 1356 | 'name': 'project_name', 1357 | 'message': 'Enter project name', 1358 | 'default': defaultProjectName, 1359 | 'validate': ProjectNameValidator 1360 | }, 1361 | ] 1362 | 1363 | answers = pyprompt(questions) 1364 | 1365 | project_name = answers["project_name"] 1366 | else: 1367 | project_name = name 1368 | 1369 | spinner = Halo(text='Generating project: ' + project_name, spinner='dots') 1370 | spinner.start() 1371 | response = api.generate_project(project_name) 1372 | settings = json.loads(response.content) 1373 | f = open(digger_yaml(), "w") 1374 | ydump(settings, f) 1375 | spinner.stop() 1376 | 1377 | print("project generated successfully") 1378 | report_async({"command": f"dg project generate"}, settings=settings, status="complete") 1379 | 1380 | 1381 | 1382 | @cli.group() 1383 | @require_auth 1384 | def service(): 1385 | """ 1386 | Configure a new service 1387 | """ 1388 | 1389 | 1390 | @service.command(name="add") 1391 | def service_add(): 1392 | action = "add" 1393 | report_async({"command": f"dg service add"}, status="start") 1394 | # service_names = get_service_names() 1395 | service_names = list(filter(lambda x: x != "digger-master" and os.path.isdir(x) and not x.startswith("."), os.listdir(os.getcwd()))) 1396 | 1397 | if len(service_names) == 0: 1398 | Bcolors.fail("No service directories found, try cloning a repo in here!") 1399 | return 1400 | 1401 | questions = [ 1402 | { 1403 | 'type': 'list', 1404 | 'name': 'service_name', 1405 | 'message': 'select repository', 1406 | 'choices': service_names 1407 | }, 1408 | ] 1409 | 1410 | answers = pyprompt(questions) 1411 | service_name = answers["service_name"] 1412 | service_key = service_name 1413 | 1414 | service_path = service_name 1415 | serviceNameOk = re.fullmatch(r'', service_name) 1416 | if not serviceNameOk: 1417 | Bcolors.warn("service names should be lowercase letters, hiphens and at most 10 characters") 1418 | service_name = transform_service_name(service_name) 1419 | Bcolors.warn(f"Updating name to: {service_name}") 1420 | 1421 | settings = get_project_settings() 1422 | 1423 | try: 1424 | dockerfile_path = find_dockerfile(service_path) 1425 | except CouldNotDetermineDockerLocation as e: 1426 | print("Could not find dockerfile in root") 1427 | dockerfile_path = dockerfile_manual_entry(service_path) 1428 | 1429 | 1430 | settings["services"] = settings.get("services", {}) 1431 | settings["services"][service_key] = { 1432 | "service_name": service_name, 1433 | "path": service_path, 1434 | "env_files": [], 1435 | "publicly_accessible": True, 1436 | "service_type": "container", 1437 | "container_port": 8080, 1438 | "health_check": "/", 1439 | "dockerfile": dockerfile_path, 1440 | "resources": {}, 1441 | "dependencies": {}, 1442 | } 1443 | 1444 | update_digger_yaml(settings) 1445 | spin(1, "Updating DGL config ... ") 1446 | 1447 | print("Service added succesfully") 1448 | report_async({"command": f"dg service add"}, settings=settings, status="complete") 1449 | 1450 | 1451 | @cli.command(name="sync") 1452 | def sync(): 1453 | """ 1454 | Sync all current services with backend 1455 | """ 1456 | settings = get_project_settings() 1457 | projectName = settings["project"]["name"] 1458 | services = settings["services"] 1459 | for key, service in services.items(): 1460 | service["name"] = service["service_name"] 1461 | servicesList = json.dumps(list(services.values())) 1462 | api.sync_services(projectName, {"services": servicesList}) 1463 | Bcolors.okgreen("digger.yml services synced with backend successfully") 1464 | 1465 | 1466 | @cli.command() 1467 | @click.argument("service_name") 1468 | # @click.argument("webapp_name") 1469 | def logs(service_name): 1470 | """ 1471 | View the logs of a service 1472 | """ 1473 | settings = get_project_settings() 1474 | projectName = settings["project"]["name"] 1475 | response = api.get_logs(projectName) 1476 | content = json.loads(response.content) 1477 | for record in content: 1478 | time = datetime.fromtimestamp(record["timestamp"]/1000).strftime("%Y-%m-%d %H:%M") 1479 | print(f'[[{time}]]', record["message"]) 1480 | 1481 | 1482 | @cli.command() 1483 | @click.argument("action") 1484 | # @click.argument("webapp_name") 1485 | def webapp(action): 1486 | """ 1487 | Configure a web application (frontend) 1488 | """ 1489 | 1490 | if action == "create": 1491 | pass 1492 | 1493 | elif action == "add": 1494 | pass 1495 | 1496 | @cli.group() 1497 | def resource(): 1498 | """ 1499 | Configure a resource 1500 | """ 1501 | 1502 | 1503 | 1504 | # exec main function if frozen binary 1505 | if getattr(sys, 'frozen', False): 1506 | try: 1507 | cli(sys.argv[1:]) 1508 | except Exception as e: 1509 | raise e 1510 | -------------------------------------------------------------------------------- /diggercli/diggerconfig.py: -------------------------------------------------------------------------------- 1 | import os 2 | from oyaml import load as yload, dump as ydump 3 | try: 4 | from yaml import CLoader as Loader, CDumper as Dumper 5 | except ImportError: 6 | from yaml import Loader, Dumper 7 | from diggercli.utils.pprint import Bcolors 8 | 9 | class Service: 10 | def __init__(self, serviceType, configpath): 11 | self.type = serviceType 12 | self.configpath = configpath 13 | 14 | class ProjectDetector: 15 | DIGGER = "digger" 16 | DOCKER = "docker" 17 | FRONTEND = "frontend" 18 | UNKNOWN = "unknown" 19 | 20 | def digger_test(self, path): 21 | files = os.listdir(path) 22 | 23 | if "digger.yml" in files: 24 | return Service(self.DIGGER, "digger.yml") 25 | return False 26 | 27 | def docker_test(self, path): 28 | files = os.listdir(path) 29 | if "dockerfile" in list(map(lambda x: x.lower(), files)): 30 | return Service(self.DOCKER, "Dockerfile") 31 | return False 32 | 33 | def javascript_test(self, path): 34 | files = os.listdir(path) 35 | if "package.json" in files: 36 | return Service(self.FRONTEND, "package.json") 37 | return False 38 | 39 | def detect_service(self, path): 40 | 41 | Bcolors.warn("... Searching for digger.yml") 42 | dgtest = self.digger_test(path) 43 | if dgtest != False: 44 | Bcolors.okgreen("digger.yml file found .. loading settings") 45 | return dgtest 46 | Bcolors.warn("[x] digger.yml not found") 47 | 48 | Bcolors.warn("... Searching for dockerfile") 49 | dockertest = self.docker_test(path) 50 | if dockertest != False: 51 | return dockertest 52 | Bcolors.warn("[x] dockerfile not found") 53 | 54 | Bcolors.warn("... Searching for package.json") 55 | jstest = self.javascript_test(path) 56 | if jstest != False: 57 | return jstest 58 | Bcolors.warn("[x] package.json not found") 59 | 60 | 61 | return Service(self.UNKNOWN, None) 62 | 63 | 64 | class Generator(): 65 | 66 | def __init__(self, services): 67 | self.state = { 68 | "version": "1.0.0", 69 | "services": {} 70 | } 71 | self.services = services 72 | self.update_state() 73 | 74 | def dump_yaml(self): 75 | f = open("digger.yml", "w") 76 | ydump(self.state, f) 77 | 78 | @classmethod 79 | def load_yaml(cls): 80 | return yload(open("digger.yml"), Loader=Loader) 81 | 82 | 83 | def update_state(self): 84 | for service in self.services: 85 | if service.type == ProjectDetector.DOCKER: 86 | self.generate_docker(service) 87 | elif service.type == ProjectDetector.FRONTEND: 88 | self.generate_frontend(service) 89 | else: 90 | Bcolors.warn(f"Unknown type {service.type}") 91 | 92 | 93 | def generate_frontend(self, service): 94 | self.state["services"]["frontend"] = { 95 | "service_name": "frontend", 96 | "root": ".", 97 | "build_cmd": "npm run build", 98 | "dist_path": "dist", 99 | "publicly_accissible": True, 100 | "packagejson": service.configpath, 101 | } 102 | 103 | def generate_docker(self, service): 104 | self.state["services"]["backend"] = { 105 | "service_name": "backend", 106 | "root": ".", 107 | # "env_files": [], 108 | "publicly_accissible": True, 109 | "type": "container", 110 | "container_port": 3000, 111 | "health_check": "/", 112 | "dockerfile": service.configpath, 113 | "resources": {}, 114 | "dependencies": {}, 115 | } 116 | 117 | -------------------------------------------------------------------------------- /diggercli/env/.env: -------------------------------------------------------------------------------- 1 | BACKEND_ENDPOINT=https://backend.digger.dev 2 | WEBAPP_ENDPOINT=https://app.digger.dev -------------------------------------------------------------------------------- /diggercli/env/.env.example: -------------------------------------------------------------------------------- 1 | BACKEND_ENDPOINT=http://localhost:8080 -------------------------------------------------------------------------------- /diggercli/env/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diggerhq/cli/dd2719bf8aaee04c431671cc92a416efc32915cd/diggercli/env/__init__.py -------------------------------------------------------------------------------- /diggercli/env/pyinstaller/.env: -------------------------------------------------------------------------------- 1 | BACKEND_ENDPOINT=https://backend.digger.dev 2 | WEBAPP_ENDPOINT=https://app.digger.dev -------------------------------------------------------------------------------- /diggercli/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class CouldNotDetermineDockerLocation(Exception): 4 | pass 5 | 6 | class ApiRequestException(Exception): 7 | pass 8 | 9 | class FileTooLargeError(Exception): 10 | pass 11 | -------------------------------------------------------------------------------- /diggercli/fileio.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import json 4 | import zipfile 5 | import time 6 | from pathlib import Path 7 | 8 | import requests 9 | import tempfile 10 | from diggercli import api 11 | from diggercli.exceptions import FileTooLargeError 12 | 13 | 14 | def download_file(url, path): 15 | # NOTE the stream=True parameter below 16 | with requests.get(url, stream=True) as r: 17 | r.raise_for_status() 18 | with open(path, 'wb') as f: 19 | for chunk in r.iter_content(chunk_size=8192): 20 | # If you have chunk encoded response uncomment if 21 | # and set chunk_size parameter to None. 22 | #if chunk: 23 | f.write(chunk) 24 | 25 | def download_terraform_files(projectName, environment, region, target, services, destinationDir): 26 | response = api.download_terraform_async(projectName, environment, region, target, services) 27 | job = json.loads(response.content) 28 | 29 | while True: 30 | jobResponse = api.terraform_generate_status(job["terraform_job_id"]) 31 | jobStatus = json.loads(jobResponse.content) 32 | 33 | status = jobStatus["status"] 34 | 35 | if status == "COMPLETED": 36 | break 37 | 38 | time.sleep(2) 39 | 40 | fileUrl = jobStatus["file_url"] 41 | 42 | tmpZipPath = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names())) 43 | download_file(fileUrl, tmpZipPath) 44 | with zipfile.ZipFile(tmpZipPath, 'r') as zip_ref: 45 | zip_ref.extractall(destinationDir) 46 | 47 | 48 | def zipdir(ziph): 49 | # ziph is zipfile handle 50 | path = os.getcwd() 51 | lenDirPath = len(path) 52 | for root, dirs, files in os.walk(path): 53 | for file in files: 54 | # if root.endswith("node_modules"): continue 55 | filePath = os.path.join(root, file) 56 | # the second argument ensures accurate tree structure in the zip file 57 | ziph.write(filePath, filePath[lenDirPath:]) 58 | 59 | def git_zip(zippath): 60 | subprocess.run([ 61 | "git", 62 | "archive", 63 | "--format", "zip", 64 | "--output", zippath, 65 | "master", 66 | ], check=True) 67 | 68 | def git_exists(): 69 | devnull = open(os.devnull, 'w') 70 | try: 71 | subprocess.run(["git", "status"], stdout=devnull, stderr=devnull, check=True) 72 | return True 73 | except subprocess.CalledProcessError as grepexc: 74 | return False 75 | 76 | def git_zip_or_zipdir(zippath): 77 | 78 | if ".git" in os.listdir() and git_exists(): 79 | git_zip(zippath) 80 | return 81 | 82 | # create the zip path 83 | ziph = zipfile.ZipFile(zippath, "w", compression=zipfile.ZIP_DEFLATED) 84 | zipdir(ziph) 85 | ziph.close() 86 | 87 | def upload_code(tmp_project_uuid, service_name): 88 | 89 | response = api.get_signed_url_for_code_upload(tmp_project_uuid, { 90 | "service_name": service_name 91 | }) 92 | content = json.loads(response.content) 93 | 94 | zippath = tempfile.mkdtemp() 95 | zippath = os.path.join(zippath, "code.zip") 96 | # create the zip path 97 | ziph = zipfile.ZipFile(zippath, "w", compression=zipfile.ZIP_DEFLATED) 98 | zipdir(ziph) 99 | ziph.close() 100 | 101 | file_size = Path(zippath).stat().st_size 102 | if file_size > 100 * 1024 * 1024: 103 | raise FileTooLargeError("Code size exceeds 100MB") 104 | 105 | with open(zippath, 'rb') as zipf: 106 | # uploading the object 107 | object_name = content["fields"]["key"] 108 | files = {'file': (object_name, zipf)} 109 | upload_response = requests.post(content['url'], data=content['fields'], files=files) 110 | print(upload_response.status_code) 111 | assert upload_response.status_code//100 == 2 112 | 113 | -------------------------------------------------------------------------------- /diggercli/projects.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from pathlib import Path 4 | from diggercli import api, diggerconfig 5 | 6 | 7 | def create_temporary_project(): 8 | settings = diggerconfig.Generator.load_yaml() 9 | services = json.dumps(settings["services"]) 10 | response = api.generate_tmp_project({ 11 | "services": services 12 | }) 13 | Path(".digger").mkdir(parents=True, exist_ok=True) 14 | response = json.loads(response.content) 15 | tmpId = response["id"] 16 | f = open(".digger/temp.json", "w") 17 | json.dump({"temp_project_id": tmpId}, f) 18 | return tmpId 19 | 20 | def get_temporary_project_id(): 21 | f = open(".digger/temp.json") 22 | content = json.load(f) 23 | return content["temp_project_id"] -------------------------------------------------------------------------------- /diggercli/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import random 4 | import sys 5 | import http.server as SimpleHTTPServer 6 | import socketserver as SocketServer 7 | from urllib.parse import urlparse 8 | from urllib.parse import parse_qs 9 | import logging 10 | import sys 11 | 12 | 13 | callback=None 14 | 15 | class GetHandler( 16 | SimpleHTTPServer.SimpleHTTPRequestHandler 17 | ): 18 | 19 | def do_GET(self): 20 | # Extract query param 21 | query_components = parse_qs(urlparse(self.path).query) 22 | 23 | if 'redirect_uri' in query_components: 24 | redirect_uri = query_components["redirect_uri"][0] 25 | else: 26 | redirect_uri = 'https://app.digger.dev/' 27 | 28 | if 'token' in query_components: 29 | token = query_components["token"][0] 30 | else: 31 | print("WARNING: token not found, aborting") 32 | return 33 | 34 | # Sending an '200 OK' response 35 | self.send_response(301) 36 | 37 | self.send_header('Location', f'{redirect_uri}') 38 | 39 | # Whenever using 'send_header', you also have to call 'end_headers' 40 | self.end_headers() 41 | 42 | if callback is not None: 43 | callback(token) 44 | 45 | print("callback token completed successfully") 46 | sys.exit() 47 | 48 | return 49 | 50 | class GetHandlerWithCallback(GetHandler): 51 | def __init__(self, callback): 52 | self.callback = callback 53 | 54 | def start_server(port, callback_fn): 55 | global callback 56 | callback = callback_fn 57 | Handler = GetHandler 58 | print(f"server listening on port {port}") 59 | httpd = SocketServer.TCPServer(("", port), Handler) 60 | 61 | httpd.serve_forever() 62 | 63 | -------------------------------------------------------------------------------- /diggercli/templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diggerhq/cli/dd2719bf8aaee04c431671cc92a416efc32915cd/diggercli/templates/__init__.py -------------------------------------------------------------------------------- /diggercli/templates/environments/local-docker/.postgres.env: -------------------------------------------------------------------------------- 1 | POSTGRES_USER={{dbuser}} 2 | POSTGRES_PASSWORD={{dbpassword}} 3 | POSTGRES_DB={{dbname}} 4 | DATABASE_URL=postgresql://{{dbuser}}:{{dbpassword}}@{{dbhost}}:5432/{{dbname}} 5 | -------------------------------------------------------------------------------- /diggercli/templates/environments/local-docker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diggerhq/cli/dd2719bf8aaee04c431671cc92a416efc32915cd/diggercli/templates/environments/local-docker/__init__.py -------------------------------------------------------------------------------- /diggercli/templates/environments/local-docker/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diggerhq/cli/dd2719bf8aaee04c431671cc92a416efc32915cd/diggercli/templates/environments/local-docker/docker-compose.yml -------------------------------------------------------------------------------- /diggercli/transformers.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | def transform_service_name(serviceName): 4 | """ 5 | transforms an invalid service name into a valid one by capping length 6 | and replacing all invalid characters 7 | """ 8 | serviceName = re.sub(r"[\ \_]", "-", serviceName) 9 | serviceName = serviceName.lower() 10 | serviceName = serviceName[:10] 11 | return serviceName 12 | -------------------------------------------------------------------------------- /diggercli/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diggerhq/cli/dd2719bf8aaee04c431671cc92a416efc32915cd/diggercli/utils/__init__.py -------------------------------------------------------------------------------- /diggercli/utils/misc.py: -------------------------------------------------------------------------------- 1 | import os 2 | from oyaml import load as yload, dump as ydump 3 | try: 4 | from yaml import CLoader as Loader, CDumper as Dumper 5 | except ImportError: 6 | from yaml import Loader, Dumper 7 | from diggercli.constants import DIGGER_CONFIG_FILE 8 | 9 | 10 | def _retry_until(callable, max_retries=60, time_between_retries=10): 11 | current_retry_count = 1 12 | while True: 13 | if current_retry_count > max_retries: 14 | break 15 | callable() 16 | current_retry_count += 1 17 | time.sleep(time_between_retries) 18 | 19 | def read_env_config_from_file(environmentName, overrideOptions={}, filePath=DIGGER_CONFIG_FILE): 20 | if not os.path.exists(filePath): 21 | return overrideOptions 22 | file = open(filePath) 23 | configOptions = yload(file, Loader=Loader) 24 | file.close() 25 | options = configOptions.get("default", {}) 26 | environmentOptions = configOptions.get(environmentName, {}) 27 | options.update(environmentOptions) 28 | options.update(overrideOptions) 29 | return options 30 | 31 | 32 | def parse_env_config_options(config : list): 33 | configOptions = {} 34 | for configOption in config: 35 | if configOption.find("=") < 0: 36 | Bcolors.error(f"each config should be of form key=val, found: {configOption}") 37 | sys.exit(-1) 38 | key,val = configOption.split("=", 1) 39 | # parse boolean inputs correctly 40 | if val.lower() == "true" or val.lower() == "false": 41 | val = (val.lower() == "true") 42 | configOptions[key] = val 43 | return configOptions 44 | 45 | # compute the variables along with overrides 46 | def compute_env_vars_with_overrides(envVars: list, servicePk: int) -> dict: 47 | res = {} 48 | # expose env variables 49 | for var in envVars: 50 | if var["service"] is None: 51 | res[var["name"]] = var["value"] 52 | 53 | # Override service parameters 54 | for var in envVars: 55 | if var["service"] == servicePk: 56 | res[var["name"]] = var["value"] 57 | 58 | return res 59 | -------------------------------------------------------------------------------- /diggercli/utils/pprint.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | from halo import Halo as TrueHalo 4 | 5 | def spin(t, msg, mode='dots'): 6 | spinner = Halo(text=msg, spinner=mode) 7 | spinner.start() 8 | time.sleep(t) 9 | spinner.stop() 10 | 11 | 12 | class FakeHalo(TrueHalo): 13 | def start(self, **kwargs): 14 | if self.text: 15 | print(self.text) 16 | 17 | def stop (self, **kwargs): pass 18 | 19 | 20 | def Halo(**kwargs): 21 | 22 | if sys.stdout.isatty(): 23 | return TrueHalo(**kwargs) 24 | else: 25 | return FakeHalo(**kwargs) 26 | 27 | 28 | class SpinnerSegment: 29 | def __init__(self, text, spinner="dots", *args, **kwargs): 30 | self.spinner = Halo(text=text, spinner=spinner) 31 | 32 | def __enter__(self): 33 | self.spinner.start() 34 | 35 | def __exit__(self, *args, **kwargs): 36 | self.spinner.stop() 37 | 38 | 39 | class Bcolors: 40 | HEADER = '\033[95m' 41 | OKBLUE = '\033[94m' 42 | OKCYAN = '\033[96m' 43 | OKGREEN = '\033[92m' 44 | OKPINK = '\033[35m' 45 | WARN = '\033[93m' 46 | FAIL = '\033[91m' 47 | ENDC = '\033[0m' 48 | BOLD = '\033[1m' 49 | UNDERLINE = '\033[4m' 50 | 51 | @classmethod 52 | def print(cls, msg, ctype): 53 | print(f"{ctype}{msg}{cls.ENDC}") 54 | 55 | @classmethod 56 | def header(cls, msg): 57 | cls.print(msg, cls.HEADER) 58 | 59 | @classmethod 60 | def okblue(cls, msg): 61 | cls.print(msg, cls.OKBLUE) 62 | 63 | @classmethod 64 | def okgreen(cls, msg): 65 | cls.print(msg, cls.OKGREEN) 66 | 67 | @classmethod 68 | def warn(cls, msg): 69 | cls.print(msg, cls.WARN) 70 | 71 | @classmethod 72 | def fail(cls, msg): 73 | cls.print(msg, cls.FAIL) 74 | 75 | @classmethod 76 | def endc(cls, msg): 77 | cls.print(msg, cls.ENDC) 78 | 79 | @classmethod 80 | def bold(cls, msg): 81 | cls.print(msg, cls.BOLD) 82 | 83 | @classmethod 84 | def underline(cls, msg): 85 | cls.print(msg, cls.UNDERLINE) -------------------------------------------------------------------------------- /diggercli/validators.py: -------------------------------------------------------------------------------- 1 | import re 2 | from PyInquirer import prompt, Validator, ValidationError 3 | from prompt_toolkit import document 4 | 5 | def project_name_validate(projectName): 6 | if len(projectName) > 10: 7 | raise ValueError('Project name should at most 10 characters') 8 | ok=re.fullmatch(r'^[a-z0-9\-]+$', projectName) 9 | if not ok: 10 | raise ValueError('Project name should only contain lowercase letters, numbers and hiphen (-)') 11 | 12 | class ProjectNameValidator(Validator): 13 | def validate(self, document: document.Document) -> None: 14 | try: 15 | project_name_validate(document.text) 16 | except ValueError as e: 17 | raise ValidationError(message=str(e), cursor_position=len(document.text)) 18 | 19 | def env_name_validate(envName): 20 | if len(envName) > 10: 21 | raise ValueError('Environment name should at most 10 characters') 22 | ok=re.fullmatch(r'^[a-z0-9\-]+$', envName) 23 | if not ok: 24 | raise ValueError('Environment name should only contain lowercase letters, numbers and hiphen (-)') 25 | -------------------------------------------------------------------------------- /docker/Dockerfile-release.debian: -------------------------------------------------------------------------------- 1 | FROM python:3.7-buster 2 | ENV PYTHONUNBUFFERED 1 3 | 4 | # Adds our application code to the image 5 | copy . /code/ 6 | WORKDIR /code/ 7 | 8 | RUN pwd 9 | 10 | RUN pip install -r src/requirements.txt && pip install pyinstaller===3.5 11 | 12 | ENTRYPOINT "docker/scripts/release.sh" 13 | -------------------------------------------------------------------------------- /docker/Dockerfile.ecr.publish: -------------------------------------------------------------------------------- 1 | FROM python:3.9.5-alpine3.13 2 | 3 | WORKDIR /code 4 | # COPY . /code 5 | 6 | ENV BACKEND_ENDPOINT="https://backend.digger.dev" 7 | ENV WEBAPP_ENDPOINT="https://app.digger.dev" 8 | ARG TAG="master" 9 | 10 | RUN apk add git gcc libcurl python3-dev libc-dev docker 11 | RUN pip install --upgrade awscli \ 12 | "git+https://github.com/diggerhq/cli@$TAG" 13 | 14 | #CMD ["/bin/sh", "-c", "/code/entrypoint.sh"] 15 | -------------------------------------------------------------------------------- /docker/scripts/release.sh: -------------------------------------------------------------------------------- 1 | pyinstaller -y src/dg.spec 2 | mkdir -p /dist 3 | cp -r dist/* /dist 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | 2 | # == installing local package to fix pyinstaller == # 3 | . 4 | -------------------------------------------------------------------------------- /scripts/buildformac.sh: -------------------------------------------------------------------------------- 1 | # cd dg 2 | python3 -m venv py3 3 | source py3/bin/activate 4 | pip install pyinstaller 5 | pip install -r requirements.txt 6 | pyinstaller --clean -y --dist ./dist/dg-mac --workpath /tmp dg.spec 7 | chmod +x dist/dg-mac/dg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | 5 | setup( 6 | name='Digger CLI', 7 | # package_dir={ 8 | # 'diggercli': '' 9 | # }, 10 | install_requires=[ 11 | "certifi==2020.6.20", 12 | "chardet==3.0.4", 13 | "click==7.1.2", 14 | "colorama==0.4.3", 15 | "docutils==0.15.2", 16 | "environs==9.0.0", 17 | "halo==0.0.30", 18 | "idna==2.10", 19 | "Jinja2==2.11.3", 20 | "jmespath==0.10.0", 21 | "log-symbols==0.0.14", 22 | "MarkupSafe==1.1.1", 23 | "marshmallow==3.9.0", 24 | "oyaml==1.0", 25 | "prompt_toolkit==1.0.14", 26 | "pyasn1==0.4.8", 27 | "Pygments==2.7.4", 28 | "PyInquirer==1.0.3", 29 | "python-dateutil==2.8.1", 30 | "python-dotenv==0.15.0", 31 | "PyYAML==5.4", 32 | "regex==2020.10.15", 33 | "requests==2.26.0", 34 | "rsa==4.7", 35 | "six==1.15.0", 36 | "spinners==0.0.24", 37 | "termcolor==1.1.0", 38 | "urllib3==1.26.5", 39 | "wcwidth==0.2.5", 40 | "boto3==1.17.11", 41 | ], 42 | version="1.0", 43 | py_modules=["diggercli",], 44 | packages=['diggercli', 'diggercli.utils'], 45 | entry_points=''' 46 | [console_scripts] 47 | dg=diggercli.dg:cli 48 | ''' 49 | ) 50 | -------------------------------------------------------------------------------- /tst/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diggerhq/cli/dd2719bf8aaee04c431671cc92a416efc32915cd/tst/__init__.py -------------------------------------------------------------------------------- /tst/test_cli.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | from unittest.mock import patch, MagicMock 4 | from click.testing import CliRunner 5 | from diggercli import dg 6 | from diggercli.dg import cli 7 | 8 | # mocking objects 9 | dg.api = MagicMock() 10 | dg.auth.require_auth = MagicMock() 11 | dg.fetch_github_token = MagicMock() 12 | dg.pyprompt = MagicMock() 13 | dg.report_async = MagicMock() 14 | dg.update_digger_yaml = MagicMock() 15 | dg.json = MagicMock() 16 | dg.create_aws_profile = MagicMock() 17 | 18 | 19 | class ClickTestMixin(): 20 | def setUp(self): 21 | self.runner = CliRunner() 22 | 23 | def _invoke_click_command(self, command): 24 | self.runner = CliRunner() 25 | with self.runner.isolated_filesystem(): 26 | result = self.runner.invoke( 27 | cli, 28 | command, 29 | catch_exceptions=False 30 | ) 31 | return result 32 | 33 | 34 | class TestProject(ClickTestMixin, unittest.TestCase): 35 | 36 | def test_project_init(self): 37 | result = self._invoke_click_command(["project", "init"]) 38 | assert not result.exception 39 | 40 | 41 | class TestService(ClickTestMixin, unittest.TestCase): 42 | 43 | def test_service_add(self): 44 | result = self._invoke_click_command(["service", "add"]) 45 | assert not result.exception 46 | 47 | 48 | @patch("diggercli.dg.get_project_settings") 49 | class TestEnv(ClickTestMixin, unittest.TestCase): 50 | 51 | def test_env_list(self, get_project_settings): 52 | get_project_settings.return_value = {"project": {"name": "project"}} 53 | result = self._invoke_click_command(["env", "list"]) 54 | assert not result.exception 55 | 56 | # def test_env_create(self, get_project_settings): 57 | # result = self._invoke_click_command(["env", "create", "prod"]) 58 | # assert not result.exception 59 | 60 | # def test_env_sync(self, get_project_settings): 61 | # result = self._invoke_click_command(["env", "sync-tform", "prod"]) 62 | # assert not result.exception 63 | 64 | # def test_env_build(self, get_project_settings): 65 | # result = self._invoke_click_command(["env", "build"]) 66 | # assert not result.exception 67 | 68 | # def test_env_push(self, get_project_settings): 69 | # result = self._invoke_click_command(["env", "push"]) 70 | # assert not result.exception 71 | 72 | # def test_env_deploy(self, get_project_settings): 73 | # result = self._invoke_click_command(["env", "deploy"]) 74 | # assert not result.exception 75 | 76 | # def test_env_destroy(self, get_project_settings): 77 | # result = self._invoke_click_command(["env", "destroy"]) 78 | # assert not result.exception 79 | 80 | def test_env_history(self, get_project_settings): 81 | result = self._invoke_click_command(["env", "history"]) 82 | assert not result.exception 83 | 84 | # def test_env_apply(self, get_project_settings): 85 | # result = self._invoke_click_command(["env", "apply", "prod"]) 86 | # assert not result.exception 87 | 88 | 89 | class TestAuth(ClickTestMixin, unittest.TestCase): 90 | def test_auth(self): 91 | result = self._invoke_click_command(["auth"]) 92 | assert not result.exception 93 | 94 | 95 | class TestLogs(ClickTestMixin, unittest.TestCase): 96 | @patch("diggercli.dg.get_project_settings") 97 | def test_logs(self, get_project_settings): 98 | result = self._invoke_click_command(["logs", "serviceName"]) 99 | assert not result.exception 100 | 101 | 102 | @patch("diggercli.dg.get_project_settings") 103 | class TestWebapp(ClickTestMixin, unittest.TestCase): 104 | 105 | def test_logs(self, get_project_settings): 106 | result = self._invoke_click_command(["webapp", "create"]) 107 | assert not result.exception 108 | 109 | def test_logs(self, get_project_settings): 110 | result = self._invoke_click_command(["webapp", "add"]) 111 | assert not result.exception 112 | 113 | 114 | 115 | if __name__ == '__main__': 116 | unittest.main() 117 | -------------------------------------------------------------------------------- /tst/test_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | import tempfile 4 | from oyaml import load as yload, dump as ydump 5 | try: 6 | from yaml import CLoader as Loader, CDumper as Dumper 7 | except ImportError: 8 | from yaml import Loader, Dumper 9 | from diggercli import dg 10 | from unittest.mock import patch, MagicMock 11 | from diggercli.utils.misc import ( 12 | read_env_config_from_file, 13 | parse_env_config_options 14 | ) 15 | 16 | 17 | class TestFileConfigOverride(unittest.TestCase): 18 | def test_without_file(self): 19 | options = {"a": "b"} 20 | expected = options 21 | found = read_env_config_from_file("staging", overrideOptions=options) 22 | self.assertDictEqual(expected, found) 23 | 24 | def test_with_file_and_no_overrides(self): 25 | config = { 26 | "default": { 27 | "a": "b", 28 | "x": "y", 29 | }, 30 | "staging": { 31 | "x": "k", 32 | }, 33 | "prod": { 34 | "x": "x", 35 | }, 36 | 37 | } 38 | with tempfile.NamedTemporaryFile("w") as f: 39 | ydump(config, f) 40 | expected = { 41 | "a": "b", 42 | "x": "k" 43 | } 44 | found = read_env_config_from_file("staging", overrideOptions={}, filePath=f.name) 45 | self.assertDictEqual(expected, found) 46 | 47 | # testing the default case 48 | expected = { 49 | "a": "b", 50 | "x": "y", 51 | } 52 | found = read_env_config_from_file("newenv", overrideOptions={}, filePath=f.name) 53 | self.assertDictEqual(expected, found) 54 | 55 | def test_with_file_and_some_config_overrides(self): 56 | options = {"a": "a"} 57 | found = read_env_config_from_file("staging", overrideOptions={}) 58 | config = { 59 | "default": { 60 | "a": "b", 61 | "x": "y", 62 | }, 63 | "staging": { 64 | "x": "k", 65 | }, 66 | "prod": { 67 | "x": "x", 68 | }, 69 | 70 | } 71 | with tempfile.NamedTemporaryFile("w") as f: 72 | ydump(config, f) 73 | expected = { 74 | "a": "a", 75 | "x": "k" 76 | } 77 | 78 | found = read_env_config_from_file("staging", overrideOptions=options, filePath=f.name) 79 | self.assertDictEqual(expected, found) 80 | 81 | 82 | class TestConfigFromCli(unittest.TestCase): 83 | 84 | def test_input_from_cli(self): 85 | expected = { 86 | "x": "y", 87 | "y": "z=123" 88 | } 89 | found = parse_env_config_options([ 90 | "x=y", 91 | "y=z=123" 92 | ]) 93 | self.assertDictEqual(expected, found) -------------------------------------------------------------------------------- /tst/test_restrieve_aws_credentials.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | import tempfile 4 | from diggercli import dg 5 | from unittest.mock import patch, MagicMock 6 | 7 | prompt_for_aws_keys_mock = MagicMock() 8 | prompt_for_aws_keys_mock.return_value = { 9 | "aws_key": "input_user_key", 10 | "aws_secret": "input_user_secret" 11 | } 12 | dg.prompt_for_aws_keys = prompt_for_aws_keys_mock 13 | 14 | tmpdirname = tempfile.TemporaryDirectory() 15 | dg.DIGGERHOME_PATH = tmpdirname.name 16 | open(os.path.join(tmpdirname.name, "credentials"), "w").close() 17 | 18 | def override_creds_file(): 19 | global tmpdirname 20 | f = open(os.path.join(tmpdirname.name, "credentials"), "w") 21 | f.write(""" 22 | [project] 23 | aws_access_key_id = file_aws_key 24 | aws_secret_access_key = file_aws_secret 25 | """) 26 | f.close() 27 | 28 | class TestAwsRetriveCredentials(unittest.TestCase): 29 | 30 | @patch("diggercli.dg.prompt_for_aws_keys") 31 | def test_retrieve_with_prompt(self, prompt_for_aws_keys): 32 | prompt_for_aws_keys.return_value = { 33 | "aws_key": "input_user_key", 34 | "aws_secret": "input_user_secret" 35 | } 36 | credentials = dg.retreive_aws_creds("project", "environment", aws_key="aws_key", aws_secret="aws_secret", prompt=True) 37 | prompt_for_aws_keys.assert_called() 38 | self.assertEqual(credentials["aws_key"], "input_user_key") 39 | self.assertEqual(credentials["aws_secret"], "input_user_secret") 40 | 41 | def test_retrieve_with_key_override(self): 42 | credentials = dg.retreive_aws_creds("project", "environment", aws_key="aws_key", aws_secret="aws_secret", prompt=False) 43 | self.assertEqual(credentials["aws_key"], "aws_key") 44 | self.assertEqual(credentials["aws_secret"], "aws_secret") 45 | 46 | def test_retrieve_with_key_override_while_file_exists_returns_override(self): 47 | override_creds_file() 48 | credentials = dg.retreive_aws_creds("project", "environment", aws_key="aws_key2", aws_secret="aws_secret2", prompt=False) 49 | self.assertEqual(credentials["aws_key"], "aws_key2") 50 | self.assertEqual(credentials["aws_secret"], "aws_secret2") 51 | 52 | def test_retrieve_without_key_override(self): 53 | override_creds_file() 54 | credentials = dg.retreive_aws_creds("project", "environment", aws_key=None, aws_secret=None, prompt=False) 55 | self.assertEqual(credentials["aws_key"], "file_aws_key") 56 | self.assertEqual(credentials["aws_secret"], "file_aws_secret") 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /tst/test_validators.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | from unittest.mock import patch, MagicMock 4 | 5 | from diggercli import dg 6 | from diggercli.validators import ( 7 | project_name_validate, 8 | env_name_validate 9 | ) 10 | 11 | 12 | # TODO: parametarise tests 13 | class TestProjectValidator(unittest.TestCase): 14 | 15 | def test_project_name_valid(self): 16 | project_name_validate("iamvalid") 17 | 18 | def test_project_name_valid2(self): 19 | project_name_validate("exactlyten") 20 | 21 | def test_project_name_valid3(self): 22 | project_name_validate("hiphen-ok") 23 | 24 | def test_project_name_valid4(self): 25 | project_name_validate("0123numsok") 26 | 27 | def test_project_name_invalid(self): 28 | with self.assertRaises(ValueError) as context: 29 | project_name_validate("cant contain spaces") 30 | 31 | def test_project_name_invalid2(self): 32 | with self.assertRaises(ValueError) as context: 33 | project_name_validate("elevenexact") 34 | 35 | def test_project_name_invalid3(self): 36 | with self.assertRaises(ValueError) as context: 37 | project_name_validate("@!@£sdf") 38 | 39 | 40 | # TODO: parametarise tests 41 | class TestEnvironmentValidator(unittest.TestCase): 42 | 43 | def test_project_name_valid(self): 44 | env_name_validate("iamvalid") 45 | 46 | def test_project_name_valid2(self): 47 | env_name_validate("exactlyten") 48 | 49 | def test_project_name_valid3(self): 50 | env_name_validate("hiphen-ok") 51 | 52 | def test_project_name_valid4(self): 53 | env_name_validate("0123numsok") 54 | 55 | def test_project_name_invalid(self): 56 | with self.assertRaises(ValueError) as context: 57 | env_name_validate("cant contain spaces") 58 | 59 | def test_project_name_invalid2(self): 60 | with self.assertRaises(ValueError) as context: 61 | env_name_validate("elevenexact") 62 | 63 | def test_project_name_invalid3(self): 64 | with self.assertRaises(ValueError) as context: 65 | env_name_validate("@!@£sdf") 66 | 67 | --------------------------------------------------------------------------------