├── .github ├── actions │ └── build │ │ └── action.yaml └── workflows │ ├── create-deb-package-on-pr.yaml │ ├── create-release.yaml │ ├── release-brew.yaml │ ├── release-deb-package.yaml │ ├── run-tests.yaml │ └── update-gh-pages.yaml ├── .gitignore ├── CHANGELOG ├── CODEOWNERS ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── THIRD-PARTY ├── bin └── validate-run ├── doc ├── LICENSE ├── README ├── bin │ ├── build-html-doc │ ├── gen-litani7 │ ├── schema-to-scdoc │ └── uniquify-header-ids ├── configure ├── src │ ├── litani7 │ │ └── litani.jinja.scdoc │ ├── man │ │ ├── litani-acquire-html-dir.scdoc │ │ ├── litani-add-job.scdoc │ │ ├── litani-dump-run.scdoc │ │ ├── litani-get-jobs.scdoc │ │ ├── litani-init.scdoc │ │ ├── litani-print-html-dir.scdoc │ │ ├── litani-release-html-dir.scdoc │ │ ├── litani-run-build.scdoc │ │ ├── litani-set-jobs.scdoc │ │ └── litani-transform-jobs.scdoc │ └── voluptuous-man │ │ ├── litani-outcome-table.json.yaml │ │ └── litani-run.json.yaml └── templates │ ├── index.jinja.html │ └── voluptuous-man.jinja.scdoc ├── examples ├── add-root-node │ ├── README │ ├── original-run.sh │ └── run-all.py ├── no-standalone-transform │ ├── README │ ├── run-1.sh │ ├── run-2.sh │ ├── run-3.sh │ └── run-all.py └── rich-output │ ├── README.md │ ├── assumptions.html │ ├── file.dat │ ├── histogram.dat │ ├── run-1.sh │ ├── run-2.sh │ ├── run-3.sh │ ├── run-all.py │ ├── scripts │ ├── fib-table.py │ ├── fib.plt │ ├── fib.py │ ├── sin-output.py │ ├── sin.plt │ └── sin.py │ └── templates │ ├── fib-table.jinja.html │ └── sin-output.jinja.html ├── lib ├── __init__.py ├── capabilities.py ├── exec.py ├── graph.py ├── init.py ├── job_outcome.py ├── jobs.py ├── litani.py ├── litani_report.py ├── ninja.py ├── ninja_syntax.py ├── output_artifact.py ├── pid_file.py ├── process.py ├── render.py ├── run_build.py ├── run_printer.py ├── util.py └── validation.py ├── litani ├── script └── release ├── templates ├── dashboard.jinja.html ├── file-list.jinja.html ├── memory-peak-box.jinja.gnu ├── memory-trace.jinja.gnu ├── outcome_table.jinja.html ├── pipeline.jinja.html ├── run-parallelism.jinja.gnu └── runtime-box.jinja.gnu └── test ├── README ├── __init__.py ├── e2e ├── README ├── run └── tests │ ├── __init__.py │ ├── custom_stages.py │ ├── cwd.py │ ├── dump_run.py │ ├── exit_rc.py │ ├── exit_rc_fail_on_pipeline_failure.py │ ├── get_jobs_multiple_jobs.py │ ├── get_jobs_no_jobs.py │ ├── get_jobs_single_job.py │ ├── graph_line_break.py │ ├── html_node.py │ ├── job_id_env.py │ ├── multiproc_dump_run.py │ ├── no_pool_serialize.py │ ├── no_pool_serialize_graph.py │ ├── no_timed_out.py │ ├── no_timed_out_timeout_ignored.py │ ├── no_timed_out_timeout_ok.py │ ├── pipeline_order.py │ ├── pool_serialize.py │ ├── pool_serialize_graph.py │ ├── set_jobs_no_jobs.py │ ├── set_jobs_overwrite.py │ ├── set_multiple_jobs.py │ ├── single_pool.py │ ├── timed_out.py │ ├── timed_out_subprocess.py │ ├── timed_out_subprocess_multi_shell.py │ ├── timed_out_subprocess_shell.py │ ├── timed_out_timeout_ignored.py │ ├── timed_out_timeout_ok.py │ ├── transform_delete_job.py │ ├── transform_modify_job.py │ ├── transform_no_change_job.py │ └── zero_pool.py ├── run └── unit ├── __init__.py ├── lockable_directory.py ├── outcome_table_decider.py ├── output_artifact.py ├── run_consistency.py └── status_parser.py /.github/actions/build/action.yaml: -------------------------------------------------------------------------------- 1 | name: Build and test Deb package for Litani 2 | description: Build and test Deb package for Litani 3 | inputs: 4 | version: 5 | description: Version of deb package 6 | required: true 7 | outputs: 8 | deb-package-path: 9 | description: Deb Package Path 10 | value: ${{steps.create_packages.outputs.deb_package}} 11 | deb-package-name: 12 | description: Deb Package Name 13 | value: ${{steps.create_packages.outputs.deb_package_name}} 14 | runs: 15 | using: composite 16 | steps: 17 | - name: Setup directory for deb package 18 | run: | 19 | echo ${{ inputs.version }} 20 | sudo apt-get update 21 | sudo apt-get install -y mandoc scdoc ninja-build 22 | mkdir -p litani-${{ inputs.version }}/{DEBIAN,usr/{bin,libexec/litani,share/{doc/litani,man/{man1,man5,man7}}}} 23 | touch litani-${{ inputs.version }}/DEBIAN/control 24 | cat << EOF > litani-${{ inputs.version }}/DEBIAN/control 25 | Package: Litani 26 | Version: ${{ inputs.version }} 27 | Architecture: amd64 28 | Depends: ninja-build, gnuplot, graphviz, python3-jinja2 29 | Maintainer: Kareem Khazem 30 | Description: Litani is a build system that provides an HTML dashboard of 31 | job results, as well as a JSON-formatted record of job results. It 32 | provides platform-independent job control (timeouts, return code control) 33 | and an output format that is easy to render into reports (for example, 34 | using the built-in renderer). 35 | EOF 36 | ./doc/configure && ninja 37 | mv bin lib templates litani litani-${{ inputs.version }}/usr/libexec/litani/ 38 | mv doc/out/man/*.1 litani-${{ inputs.version }}/usr/share/man/man1 39 | mv doc/out/man/*.5 litani-${{ inputs.version }}/usr/share/man/man5 40 | mv doc/out/man/*.7 litani-${{ inputs.version }}/usr/share/man/man7 41 | mv doc/out/html/index.html litani-${{ inputs.version }}/usr/share/doc/litani 42 | ln -s /usr/libexec/litani/litani litani-${{ inputs.version }}/usr/bin/ 43 | rm -r $(ls -A | grep -v litani-${{ inputs.version }}) 44 | shell: bash 45 | - name: Create .deb package 46 | id: create_packages 47 | run: | 48 | sudo dpkg-deb --build --root-owner-group litani-${{ inputs.version }} 49 | deb_package_name="$(ls *.deb)" 50 | echo "::set-output name=deb_package::$deb_package_name" 51 | echo "::set-output name=deb_package_name::$deb_package_name" 52 | shell: bash 53 | - name: Install Litani using deb package 54 | run: sudo apt-get update && sudo apt install -y ./litani-${{ inputs.version }}.deb 55 | shell: bash 56 | - name: Test deb package 57 | run: | 58 | litani -h 59 | man litani 60 | litani init --project-name test 61 | litani add-job --command '/usr/bin/true' --pipeline-name 'test' --ci-stage test 62 | litani run-build 63 | shell: bash 64 | -------------------------------------------------------------------------------- /.github/workflows/create-deb-package-on-pr.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | types: [opened, synchronize, reopened, labeled, unlabeled] 4 | 5 | name: Upload deb package to artifacts 6 | jobs: 7 | ubuntu-20_04-package: 8 | if: "contains(github.event.pull_request.labels.*.name, 'create-deb-package')" 9 | name: Generate ubuntu-20.04 debian package on PR 10 | runs-on: ubuntu-20.04 11 | env: 12 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Get Version 16 | run: | 17 | echo "VERSION=$(./litani -V)" >> $GITHUB_ENV 18 | - name: Build Deb Package 19 | id: build_deb_package 20 | uses: ./.github/actions/build 21 | with: 22 | version: ${{ env.VERSION }} 23 | - name: Upload report as artifact 24 | uses: actions/upload-artifact@main 25 | with: 26 | name: ${{ env.VERSION }}-${{ runner.os }}-deb-package 27 | path: ${{ steps.build_deb_package.outputs.deb-package-path }} 28 | -------------------------------------------------------------------------------- /.github/workflows/create-release.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | tags: 4 | - '*' 5 | 6 | name: Create Release 7 | 8 | jobs: 9 | perform-release: 10 | name: Perform Release 11 | runs-on: ubuntu-20.04 12 | env: 13 | GITHUB_TOKEN: ${{ secrets.RELEASE_CI_ACCESS_TOKEN }} 14 | steps: 15 | - name: Checkout code 16 | uses: actions/checkout@v2 17 | - name: Get Version 18 | run: echo "VERSION=${GITHUB_REF/refs\/tags\/}" >> $GITHUB_ENV 19 | - name: Create release 20 | uses: actions/create-release@v1 21 | with: 22 | tag_name: ${{ env.VERSION }} 23 | release_name: ${{ env.VERSION }} 24 | body: | 25 | This is Litani version ${{ env.VERSION }}. 26 | 27 | ## MacOS 28 | 29 | On MacOS, install Litani using [Homebrew](https://brew.sh/) with 30 | 31 | ```sh 32 | brew install litani 33 | ``` 34 | 35 | or upgrade (if it's already been installed) with: 36 | 37 | ```sh 38 | brew upgrade litani 39 | ``` 40 | 41 | ## Ubuntu 42 | 43 | On Ubuntu, install Litani by downloading the *.deb package below for ubuntu-20.04 and install with 44 | 45 | ```sh 46 | # Ubuntu 20.04: 47 | $ apt install -y ./litani-${{ env.VERSION }}.deb 48 | ``` 49 | draft: false 50 | prerelease: false 51 | -------------------------------------------------------------------------------- /.github/workflows/release-brew.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: [created] 4 | 5 | name: Release to brew 6 | jobs: 7 | homebrew-pr: 8 | name: Homebrew Bump Formula PR 9 | runs-on: macos-10.15 10 | steps: 11 | - name: Get release tag name 12 | run: echo "RELEASE_TAG=${GITHUB_REF/refs\/tags\/}" >> $GITHUB_ENV 13 | - name: Configure git user name and email 14 | uses: Homebrew/actions/git-user-config@master 15 | with: 16 | username: aws-build-accumulator-release-ci 17 | - name: Create homebrew PR 18 | run: | 19 | brew update-reset 20 | brew bump-formula-pr --tag "$RELEASE_TAG" --revision "$GITHUB_SHA" litani 21 | env: 22 | HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.RELEASE_CI_ACCESS_TOKEN }} 23 | -------------------------------------------------------------------------------- /.github/workflows/release-deb-package.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: [created] 4 | 5 | name: Release deb package 6 | jobs: 7 | ubuntu-20_04-package: 8 | name: Generate ubuntu-20.04 debian package 9 | runs-on: ubuntu-20.04 10 | env: 11 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Get Version 15 | run: echo "VERSION=${GITHUB_REF/refs\/tags\/}" >> $GITHUB_ENV 16 | - name: Build Deb Package 17 | id: build_deb_package 18 | uses: ./.github/actions/build 19 | with: 20 | version: ${{ env.VERSION }} 21 | - name: Upload release binary 22 | uses: actions/upload-release-asset@v1.0.2 23 | with: 24 | upload_url: ${{ github.event.release.upload_url }} 25 | asset_path: ${{ steps.build_deb_package.outputs.deb-package-path }} 26 | asset_name: ${{ steps.build_deb_package.outputs.deb-package-name }} 27 | asset_content_type: application/x-deb 28 | -------------------------------------------------------------------------------- /.github/workflows/run-tests.yaml: -------------------------------------------------------------------------------- 1 | name: Run Litani Tests 2 | on: 3 | pull_request: 4 | types: [opened, synchronize, reopened, labeled, unlabeled] 5 | push: 6 | branches: [ release, develop ] 7 | 8 | jobs: 9 | test-litani: 10 | if: "!contains(github.event.pull_request.labels.*.name, 'no-test')" 11 | name: Run Litani Tests 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Install dependencies 16 | run: | 17 | sudo apt-get install -y ninja-build 18 | python3 -m pip install jinja2 19 | 20 | - name: Run Unit and e2e tests 21 | run: | 22 | ./test/run 23 | exit $? 24 | 25 | test-build-documentation: 26 | name: Test Build Documentation 27 | runs-on: ubuntu-latest 28 | steps: 29 | - uses: actions/checkout@v3 30 | - name: Install dependencies 31 | run: | 32 | sudo apt-get update 33 | sudo apt-get install -y mandoc scdoc ninja-build 34 | 35 | - name: Test Documentation 36 | run: | 37 | ./doc/configure 38 | ninja 39 | -------------------------------------------------------------------------------- /.github/workflows/update-gh-pages.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: [created] 4 | 5 | name: Update documentation 6 | jobs: 7 | update-documentation: 8 | name: Update gh-pages 9 | runs-on: macos-10.15 10 | env: 11 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v2 15 | - name: Install utilities 16 | run: | 17 | brew install scdoc mandoc coreutils ninja 18 | pip3 install pyyaml jinja2 19 | - name: Build doc 20 | run: ./doc/configure && ninja 21 | - name: Publish Documentation 22 | uses: JamesIves/github-pages-deploy-action@4.1.4 23 | with: 24 | branch: gh-pages 25 | folder: doc/out/html 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | test/output 2 | 3 | .litani_cache_dir 4 | 5 | doc/tmp 6 | doc/out 7 | 8 | examples/rich-output/*.csv 9 | 10 | __pycache__/ 11 | 12 | build.ninja 13 | 14 | tmp 15 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @karkhaz 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | Thank you for contributing to Litani! This document collects some coding and 5 | process guidelines. 6 | 7 | 8 | ### HTML Dashboard 9 | 10 | - Please test your changes with both light and dark mode, and with a range of 11 | browser widths. 12 | - Almost all top-level divs should have an id attribute; this makes it easy to 13 | link to specific information. 14 | - We prefer to inline all assets (CSS, images) onto the page so that it's easy 15 | to send single, self-contained pages around. For this reason, please try to 16 | keep SVGs small. 17 | 18 | 19 | ## Releases 20 | 21 | Major releases are done through the `scripts/release` script. Patch releases are 22 | still manual. 23 | 24 | ### Creating a patch release 25 | 26 | #### Summary 27 | 28 | - Write the patch on top of `develop` as normal, and open a PR to get it merged 29 | into `develop`. 30 | - Once it's merged into `develop`, cherry-pick the commits onto a new branch 31 | called `backport`. 32 | - Add a commit to `backport`, on top of the fixes, that updates the changelog 33 | with the fix information. 34 | - Merge `release` into `backport` using a merge commit. 35 | 36 | #### Process 37 | 38 | Suppose you’re in the current state: 39 | 40 | ``` 41 | ; git log --graph --decorate --date=relative --format=format:'%C(green)%h%C(yellow) %s%C(reset)%w(0,6,6)%C(bold green)\n %C(bold red)%aN%C(reset) %cr%C(reset)%w(0,0,0)\n%-D\n%C(reset)' --all 42 | 43 | * c59fc53 add foobar 44 | | HEAD -> develop 45 | | 46 | * 139191f implement feature 47 | | 48 | * d6f2fa0 Release 1.2.0 49 | |\ tag: 1.2.0, release 50 | | | 51 | | * f3ff462 add script 52 | ``` 53 | 54 | To add a fix to the `release` branch, first create it on top of `develop`. Go 55 | through the code review process to get it merged into develop as normal. 56 | 57 | ``` 58 | ; git log --graph --decorate --date=relative --format=format:'%C(green)%h%C(yellow) %s%C(reset)%w(0,6,6)%C(bold green)\n %C(bold red)%aN%C(reset) %cr%C(reset)%w(0,0,0)\n%-D\n%C(reset)' --all 59 | 60 | * bbb222 Create two-part fix 61 | | HEAD -> develop 62 | |\ 63 | | * c59fc53 Important Fix Part 2 64 | | | 65 | | * ef45678 Important Fix Part 1 66 | |/ 67 | * abcd123 add foobar 68 | | 69 | * 139191f implement feature 70 | | 71 | * d6f2fa0 Release 1.2.0 72 | |\ tag: 1.2.0, release 73 | | | 74 | ``` 75 | 76 | Then check out `release`; check out a new branch called `backport`; and 77 | cherry-pick only the commits you need onto the `backport` branch. You can use 78 | git-log to check which commits you’re about to transplant. If there are 79 | conflicts, you will need to resolve and commit them one at a time until the 80 | graph looks the way you expect (below). 81 | 82 | ``` 83 | ; git log --pretty=%h c59fc53..develop 84 | abcd123 85 | ef45678 86 | ; git checkout release 87 | ; git checkout -b backport 88 | ; git cherry-pick c59fc53..develop 89 | ; git log --graph --decorate --date=relative --format=format:'%C(green)%h%C(yellow) %s%C(reset)%w(0,6,6)%C(bold green)\n %C(bold red)%aN%C(reset) %cr%C(reset)%w(0,0,0)\n%-D\n%C(reset)' --all 90 | 91 | * αβγ321 Important Fix Part 2 92 | | HEAD -> backport 93 | | 94 | * zyx765 Important Fix Part 1 95 | | 96 | | * bbb222 Create two-part fix 97 | | | develop 98 | | |\ 99 | | | * c59fc53 Important Fix Part 2 100 | | | | 101 | | | * ef45678 Important Fix Part 1 102 | | |/ 103 | | * c59fc53 add foobar 104 | | | 105 | | * 139191f implement feature 106 | | | 107 | |/ 108 | * d6f2fa0 Release 1.2.0 109 | |\ tag: 1.2.0, release 110 | | | 111 | ``` 112 | 113 | Finally, merge the release branch into `backport`. Currently, doing a major 114 | release after having done a minor one might require manual tweaking. 115 | 116 | ``` 117 | ; git checkout release 118 | ; git merge --no-ff backport 119 | ; git log --graph --decorate --date=relative --format=format:'%C(green)%h%C(yellow) %s%C(reset)%w(0,6,6)%C(bold green)\n %C(bold red)%aN%C(reset) %cr%C(reset)%w(0,0,0)\n%-D\n%C(reset)' --all 120 | 121 | * fff000 Bump version to 1.3.0 122 | | HEAD -> release, backport, tag: 1.3.0 123 | |\ 124 | | * αβγ321 Important Fix Part 2 125 | | | 126 | | * zyx765 Important Fix Part 1 127 | | | 128 | | | * ef45678 Important Fix Part 2 129 | | | | develop 130 | | | | 131 | | | * abcd123 Important Fix Part 1 132 | | | | 133 | | | * c59fc53 add foobar 134 | | | | 135 | | | * 139191f implement feature 136 | | | | 137 | | | / 138 | |/ / 139 | | / 140 | |/ 141 | * d6f2fa0 Release 1.2.0 142 | |\ tag: 1.2.0 143 | | | 144 | ``` 145 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Litani 2 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Litani 2 | = 3 | 4 | Litani is a build system that provides an HTML dashboard of job results, as well 5 | as a JSON-formatted record of job results. It provides platform-independent job 6 | control (timeouts, return code control) and an output format that is easy to 7 | render into reports (for example, using the built-in renderer). 8 | 9 | ### Documentation 10 | 11 | Hosted [here](https://awslabs.github.io/aws-build-accumulator/). The Homebrew 12 | and Debian packages also install the man pages locally, try 13 | `man litani-add-job`. 14 | 15 | 16 | ### Installation 17 | 18 | To install with [Homebrew](https://brew.sh): 19 | 20 | ```bash 21 | brew install litani 22 | ``` 23 | 24 | On Ubuntu, Install Litani by downloading the `*.deb` package built by each release, 25 | available on the 26 | [releases](https://github.com/awslabs/aws-build-accumulator/releases) page and 27 | run 28 | 29 | ```bash 30 | apt install -y litani-x.xx.x.deb # where x.xx.x is the release version. 31 | ``` 32 | 33 | 34 | ### Dependencies 35 | 36 | If you are cloning the source code, you will need the following dependencies: 37 | 38 | #### Required 39 | 40 | * Python3 41 | * [Ninja](https://ninja-build.org/) 42 | * `apt-get install ninja-build`, `brew install ninja` 43 | * [Jinja](https://jinja.palletsprojects.com/en/2.11.x/) 44 | * `pip3 install jinja2` 45 | 46 | #### Recommended 47 | 48 | * [Graphviz DOT](https://graphviz.org/) 49 | * `apt-get install graphviz`, `brew install graphviz` 50 | * [Gnuplot](http://www.gnuplot.info/) to generate graphs on the dashboard 51 | * `apt-get install gnuplot`, `brew install gnuplot` 52 | 53 | #### Optional 54 | 55 | * [Voluptuous](https://pypi.org/project/voluptuous/) to perform 56 | sanity-checking on internal data structures 57 | * `pip3 install voluptuous` 58 | * [coreutils](https://www.gnu.org/software/coreutils/) to build the documentation 59 | * `brew install coreutils` 60 | * [mandoc](https://mandoc.bsd.lv/) to build the documentation 61 | * `brew install mandoc` 62 | * [scdoc](https://git.sr.ht/~sircmpwn/scdoc) to build the documentation 63 | * `brew install scdoc` 64 | -------------------------------------------------------------------------------- /bin/validate-run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). 6 | # You may not use this file except in compliance with the License. 7 | # A copy of the License is located at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed 12 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | # express or implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | 16 | 17 | import argparse 18 | import json 19 | 20 | import lib.validation 21 | 22 | 23 | def main(): 24 | pars = argparse.ArgumentParser() 25 | for arg in [{ 26 | "flags": ["run_file"], 27 | }]: 28 | flags = arg.pop("flags") 29 | pars.add_argument(*flags, **arg) 30 | args = pars.parse_args() 31 | 32 | with open(args.run_file) as handle: 33 | run = json.load(handle) 34 | 35 | lib.validation.validate_run(run) 36 | 37 | 38 | if __name__ == "__main__": 39 | main() 40 | -------------------------------------------------------------------------------- /doc/README: -------------------------------------------------------------------------------- 1 | The documentation is made available under the Creative Commons 2 | Attribution-ShareAlike 4.0 International License. See the LICENSE file. 3 | -------------------------------------------------------------------------------- /doc/bin/build-html-doc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). 6 | # You may not use this file except in compliance with the License. 7 | # A copy of the License is located at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed 12 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | # express or implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | 16 | 17 | import argparse 18 | import pathlib 19 | import re 20 | 21 | import jinja2 22 | 23 | 24 | CHAPTER_PAT_STR = r'.+"head-ltitle"\>[-\.\w]+\((?P\d)\)' 25 | CHAPTER_PAT = re.compile(CHAPTER_PAT_STR) 26 | 27 | 28 | def get_args(): 29 | pars = argparse.ArgumentParser() 30 | for arg in [{ 31 | "flags": ["--html-manuals"], 32 | "nargs": "+", 33 | "required": True, 34 | "type": pathlib.Path, 35 | }, { 36 | "flags": ["--man-html-dir"], 37 | "required": True, 38 | "type": pathlib.Path, 39 | }, { 40 | "flags": ["--template-dir"], 41 | "required": True, 42 | "type": pathlib.Path, 43 | }, { 44 | "flags": ["--out-file"], 45 | "required": True, 46 | "type": pathlib.Path, 47 | }]: 48 | flags = arg.pop("flags") 49 | pars.add_argument(*flags, **arg) 50 | return pars.parse_args() 51 | 52 | 53 | def get_manual(man, man_html_dir): 54 | record = { 55 | "title": re.sub("litani-", "litani ", man.stem), 56 | "anchor": man.stem, 57 | "body": [], 58 | "chapter": get_chapter(man_html_dir / man.name), 59 | } 60 | if record["title"] == "litani.7": 61 | record["title"] = "litani" 62 | with open(man) as handle: 63 | for line in handle: 64 | record["body"].append(line.rstrip()) 65 | return record 66 | 67 | 68 | def get_chapter(man_html): 69 | with open(man_html) as handle: 70 | for line in handle: 71 | m = CHAPTER_PAT.match(line) 72 | if m: 73 | return int(m["chap"]) 74 | raise UserWarning( 75 | f"No line of man html output '{man_html}' matched chapter pattern " 76 | f"'{CHAPTER_PAT_STR}'") 77 | 78 | 79 | def get_manuals(html_manuals, man_html_dir): 80 | ret = [] 81 | for man in html_manuals: 82 | ret.append(get_manual(man, man_html_dir)) 83 | return ret 84 | 85 | 86 | def main(): 87 | args = get_args() 88 | env = jinja2.Environment( 89 | loader=jinja2.FileSystemLoader(str(args.template_dir)), 90 | autoescape=jinja2.select_autoescape( 91 | enabled_extensions=('html'), 92 | default_for_string=True)) 93 | manuals = get_manuals(args.html_manuals, args.man_html_dir) 94 | 95 | templ = env.get_template("index.jinja.html") 96 | page = templ.render(manuals=manuals) 97 | 98 | with open(args.out_file, "w") as handle: 99 | print(page, file=handle) 100 | 101 | 102 | 103 | if __name__ == "__main__": 104 | main() 105 | -------------------------------------------------------------------------------- /doc/bin/gen-litani7: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). 6 | # You may not use this file except in compliance with the License. 7 | # A copy of the License is located at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed 12 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | # express or implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | 16 | 17 | import argparse 18 | import pathlib 19 | import re 20 | 21 | import jinja2 22 | 23 | 24 | _CHAPTER_DESCRIPTIONS = { 25 | "1": "Executable Commands", 26 | "5": "File Formats & Conventions", 27 | "7": "Miscellaneous", 28 | } 29 | 30 | 31 | def _get_args(): 32 | pars = argparse.ArgumentParser() 33 | for arg in [{ 34 | "flags": ["--src"], 35 | "required": True, 36 | "type": pathlib.Path, 37 | }, { 38 | "flags": ["--dst"], 39 | "required": True, 40 | "type": pathlib.Path, 41 | }, { 42 | "flags": ["--man-dir"], 43 | "required": True, 44 | "type": pathlib.Path, 45 | }]: 46 | flags = arg.pop("flags") 47 | pars.add_argument(*flags, **arg) 48 | return pars.parse_args() 49 | 50 | 51 | def get_metadata(man): 52 | pat = re.compile(r"^litani (?P[^\s]+)\s+\-\s+(?P.+)") 53 | with open(man) as handle: 54 | for line in handle: 55 | m = pat.match(line) 56 | if m: 57 | d = m["description"] 58 | description = d[0].lower() + d[1:] 59 | 60 | # scdoc inserts these to avoid line breaks 61 | name = re.sub(r"\&", "", m["name"]) 62 | return { 63 | "name": name, 64 | "description": description, 65 | } 66 | raise UserWarning( 67 | f"Could not extract name and description from manual {man}") 68 | 69 | 70 | def _add_mans(man_dir, mans_by_chapter): 71 | for man in man_dir.iterdir(): 72 | if man.name == "litani.7": 73 | continue 74 | chapter = man.suffix[-1] 75 | mans_by_chapter[chapter].append(get_metadata(man)) 76 | for _, mans in mans_by_chapter.items(): 77 | mans.sort(key=(lambda m: m["name"])) 78 | 79 | 80 | def main(): 81 | args = _get_args() 82 | mans_by_chapter = {chapter: [] for chapter in _CHAPTER_DESCRIPTIONS} 83 | 84 | _add_mans(args.man_dir, mans_by_chapter) 85 | 86 | chapters = [] 87 | for num, mans in mans_by_chapter.items(): 88 | chapters.append({ 89 | "name": f"Chapter {num}: {_CHAPTER_DESCRIPTIONS[num]}", 90 | "mans": mans 91 | }) 92 | chapters.sort(key=(lambda c: c["name"])) 93 | 94 | env = jinja2.Environment( 95 | loader=jinja2.FileSystemLoader(str(args.src.parent)), 96 | autoescape=jinja2.select_autoescape( 97 | enabled_extensions=("html"), 98 | default_for_string=True)) 99 | templ = env.get_template(args.src.name) 100 | page = templ.render(chapters=chapters) 101 | 102 | with open(args.dst, "w") as handle: 103 | print(page, file=handle) 104 | 105 | 106 | if __name__ == "__main__": 107 | main() 108 | -------------------------------------------------------------------------------- /doc/bin/uniquify-header-ids: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sed -f 2 | # 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). 6 | # You may not use this file except in compliance with the License. 7 | # A copy of the License is located at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed 12 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | # express or implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | 16 | 17 | # Give the IDs of section headers a @MAN_NAME@ prefix. This is intended to be further filtered through a script that replaces @MAN_NAME@ with the name of a manual page. 18 | 19 | s/class="Sh" id="\([a-zA-Z_0-9]*\)">
189 |
190 | 191 |
192 | 193 | 194 | -------------------------------------------------------------------------------- /templates/memory-peak-box.jinja.gnu: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # or in the "license" file accompanying this file. This file is 11 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 12 | # ANY KIND, either express or implied. See the License for the specific 13 | # language governing permissions and limitations under the License. 14 | -#} 15 | 16 | $data << EOD 17 | {% for job in jobs -%} 18 | 0.4 {{ job["memory_trace"]["peak"]["rss"] }} {{ job["wrapper_arguments"]["pipeline_name"] }} 19 | {% endfor %}{# job in jobs #} 20 | EOD 21 | 22 | set terminal svg noenhanced size 400,800 23 | 24 | set format y '%.1s%cB' 25 | 26 | set border 2 linecolor "#263238" 27 | set ytics nomirror tc "#263238" font "Helvetica,14" 28 | unset xtics 29 | unset key 30 | 31 | set title "Peak resident memory for {{ group_name }}" tc "#263238" font "Helvetica,14" 32 | 33 | set xrange [0:1] 34 | 35 | set boxwidth 0.2 36 | 37 | plot '$data' using (0.2):2 with boxplot lc "#263238", \ 38 | '' using 1:2:3 with labels left tc "#263238" font "Helvetica,10" 39 | -------------------------------------------------------------------------------- /templates/memory-trace.jinja.gnu: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # or in the "license" file accompanying this file. This file is 11 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 12 | # ANY KIND, either express or implied. See the License for the specific 13 | # language governing permissions and limitations under the License. 14 | -#} 15 | 16 | $data << EOD 17 | {% for sample in job["memory_trace"]["trace"] -%} 18 | {{ sample["time"] }} {{ sample["rss"] }} {{ sample["vsz"] }} 19 | {% endfor %}{# sample in job["memory_trace"]["trace"] #} 20 | EOD 21 | 22 | set terminal svg noenhanced size 565,96 23 | 24 | set border 2 linecolor "#263238" 25 | 26 | set log y 10 27 | set format y "%.1s%cB" 28 | set ytics nomirror tc "#263238" font "Helvetica,9" 29 | 30 | set xdata time 31 | set timefmt "%Y-%m-%dT%H:%M:%SZ" 32 | unset xtics 33 | unset key 34 | 35 | set style line 100 lt 1 lc "#00000000" lw 2 36 | set style line 101 lt 1 lc rgb "#ee000000" lw 1 37 | set grid mxtics mytics ls 100, ls 101 38 | 39 | plot '$data' using 1:2 with lines lc "#ab47bc", \ 40 | '' using 1:3 with lines lc "#4caf50" 41 | -------------------------------------------------------------------------------- /templates/outcome_table.jinja.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # or in the "license" file accompanying this file. This file is 11 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 12 | # ANY KIND, either express or implied. See the License for the specific 13 | # language governing permissions and limitations under the License. 14 | -#} 15 | 16 | 17 | 18 | 19 | 20 | 21 | Outcome table 22 | 23 | 151 | 152 | 153 |
154 | 155 |
156 |

157 | Outcome Table 158 |

159 |

160 | Litani CI Dashboard 161 |

162 |
163 | 164 | {% if "comment" in table %} 165 |

166 | {{ table["comment"] }} 167 |

168 | {% endif %}{# "comment" in table #} 169 | 170 |
171 | 172 | {% for outcome in table["outcomes"] %} 173 |
174 | 175 | {% if outcome["type"] == "return-code" %} 176 |
177 |
178 |

return code

179 |
180 |
181 |

{{ outcome["value"] }}

182 |
183 |
184 | 185 | {% elif outcome["type"] == "timeout" %} 186 |
187 |
188 |

time

189 |
190 |
191 |

out

192 |
193 |
194 | 195 | {% elif outcome["type"] == "wildcard" %} 196 |
197 |
198 |

everything

199 |
200 |
201 |

else

202 |
203 |
204 | {% endif %}{# outcome["type"] == "return-code" #} 205 |
206 | 207 |
208 |
209 | 210 |
211 | {% if "comment" in outcome %} 212 |

{{ outcome["comment"] }}

213 | {% endif %}{# "comment" in outcome #} 214 |
215 | 216 | {% endfor %}{# outcome in table["outcomes"] #} 217 | 218 |
219 | 220 | 221 |
222 | 223 | 224 | -------------------------------------------------------------------------------- /templates/run-parallelism.jinja.gnu: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # or in the "license" file accompanying this file. This file is 11 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 12 | # ANY KIND, either express or implied. See the License for the specific 13 | # language governing permissions and limitations under the License. 14 | -#} 15 | 16 | $data << EOD 17 | {% for sample in trace -%} 18 | {{ sample["time"] }} {{ sample["running"] }} {{ n_proc }} {{ n_proc + 0.5 }} {% if loop.last %} "# cores: {{ n_proc}}" {% endif %} 19 | {% endfor %}{# sample in trace #} 20 | EOD 21 | 22 | set terminal svg noenhanced size 720,320 23 | 24 | set border 3 linecolor "#263238" 25 | 26 | set ylabel "# parallel jobs" 27 | 28 | set ytics nomirror tc "#263238" font "Helvetica,9" 29 | set xtics nomirror 30 | 31 | set xdata time 32 | set timefmt "%Y-%m-%dT%H:%M:%SZ" 33 | set format x "%H:%M:%S" 34 | unset key 35 | 36 | plot '$data' using 1:2 with lines lc "#ab47bc", \ 37 | '' using 1:3 with lines lc "#cc0000", \ 38 | '' using 1:4:5 with labels tc "#cc0000" 39 | -------------------------------------------------------------------------------- /templates/runtime-box.jinja.gnu: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # or in the "license" file accompanying this file. This file is 11 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 12 | # ANY KIND, either express or implied. See the License for the specific 13 | # language governing permissions and limitations under the License. 14 | -#} 15 | 16 | $data << EOD 17 | {% for job in jobs -%} 18 | 0.4 {{ job["duration"] }} {{ job["wrapper_arguments"]["pipeline_name"] }} 19 | {% endfor %}{# job in jobs #} 20 | EOD 21 | 22 | set terminal svg noenhanced size 400,800 23 | 24 | set border 2 linecolor "#263238" 25 | set ytics nomirror tc "#263238" font "Helvetica,14" 26 | unset xtics 27 | unset key 28 | 29 | set ylabel "seconds" tc "#263238" font "Helvetica,14" 30 | 31 | set title "Runtime for {{ group_name }}" tc "#263238" font "Helvetica,14" 32 | 33 | set xrange [0:1] 34 | 35 | set boxwidth 0.2 36 | 37 | plot '$data' using (0.2):2 with boxplot lc "#263238", \ 38 | '' using 1:2:3 with labels left tc "#263238" font "Helvetica,10" 39 | -------------------------------------------------------------------------------- /test/README: -------------------------------------------------------------------------------- 1 | Litani Test Suite 2 | ================= 3 | 4 | The `run` script uses litani to run a variety of unit and system tests. 5 | The dashboard showing the output of the latest test run will always be 6 | symlinked from test/output/latest/html/index.html. 7 | 8 | # OPTIONS 9 | 10 | *--output-dir* 11 | Litani will write all of its test output files for this run to this path 12 | 13 | *--fast* 14 | Do not run tests with SLOW variable set to True 15 | 16 | ## Writing Tests 17 | 18 | Each test is made up of 4 jobs: init, add-jobs, run-build, and check-run 19 | If --fast flag is passed, each job is given a timeout of 10 seconds. 20 | In the worst case, a test would take 40 seconds to run. 21 | For each test, SLOW is set to True if jobs fail to pass the above timeout. 22 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/aws-build-accumulator/8002c240ef4f424039ed3cc32e076c0234d01768/test/__init__.py -------------------------------------------------------------------------------- /test/e2e/README: -------------------------------------------------------------------------------- 1 | Litani End-to-End Test Suite 2 | ============================ 3 | 4 | 5 | Each file in the `tests` subdirectory contains a test definition. The `run` 6 | script imports a single one of those files, and uses the test definition to 7 | execute a complete Litani run. Once the run completes, the `run` script then 8 | loads the resulting `run.json` file and checks that it conforms to the test 9 | definition. 10 | 11 | Each "test definition" is a Python module that is expected to contain certain 12 | methods; the `run` script calls into each of these methods in order to set up, 13 | execute, and check the Litani run. Those methods are as follows: 14 | 15 | 16 | get_init_args() 17 | Returns a dict containing command-line arguments to be passed to `litani 18 | init`. The dict must contain two keys, `args` and `kwargs`. `args` is a 19 | list of switches (command-line options that don't take an argument), 20 | while `kwargs` is a dict mapping command-line options to the value(s) 21 | that they take as an argument. The keys or list items should be equal to 22 | the name of the command-line switch, without the leading `--`. Values of 23 | the `kwargs` dict can be a string or a list of strings. If a list of 24 | strings, these will be joined together with whitespace. 25 | 26 | For example, to make the run script run 27 | 28 | `litani init --pools foo:1 bar:2 --project-name baz` 29 | 30 | this method could return the dict 31 | 32 | { 33 | "kwargs": { 34 | "pools": [ "foo:1", "bar:2" ], 35 | "project-name": "baz", 36 | } 37 | } 38 | 39 | The `kwargs` dict must not contain "output-symlink", 40 | "output-directory", or "output-prefix", as the `run` script will 41 | supply its own value for these. 42 | 43 | 44 | get_jobs() 45 | Returns a list of jobs that will be added to Litani. Each job is similar 46 | to the return of `get_init_args()` above, i.e. it's a dict containing 47 | command-line arguments. The run script will add each job before 48 | executing the run. For example, to make the run script run 49 | 50 | `litani add-job --command true --interleave-stdout-stderr` 51 | `litani add-job --command false --description "oh no!"` 52 | 53 | this method should return 54 | 55 | [{ 56 | "args": [ "interleave-stdout-stderr" ], 57 | "kwargs": { "command": "true" } 58 | }, { 59 | "kwargs": { "command": "false", "description": "oh no!" } 60 | }] 61 | 62 | 63 | def get_run_build_args() 64 | Returns a dict containing the arguments to `litani run-build`, in the 65 | same format as `get_init_args()` above. 66 | 67 | 68 | check_run(run: dict) 69 | Takes a deserialized run.json file and returns True iff the Litani run 70 | was as expected. This method can check that particular keys of the run 71 | have certain values, etc. 72 | -------------------------------------------------------------------------------- /test/e2e/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). 6 | # You may not use this file except in compliance with the License. 7 | # A copy of the License is located at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed 12 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | # express or implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | 16 | 17 | import argparse 18 | import importlib 19 | import json 20 | import logging 21 | import os 22 | import pathlib 23 | import re 24 | import subprocess 25 | import sys 26 | 27 | 28 | DESCRIPTION = "Execute a single Litani run and test the resulting run.json" 29 | EPILOG = "See test/e2e/README for the organization of this test suite" 30 | 31 | 32 | def run_cmd(cmd, check): 33 | cmd_list = [str(c) for c in cmd] 34 | print(" ".join(cmd_list)) 35 | try: 36 | proc = subprocess.run(cmd_list, check=check) 37 | except subprocess.CalledProcessError: 38 | logging.error("Invocation failed") 39 | sys.exit(1) 40 | except FileNotFoundError: 41 | logging.error("Executable not found") 42 | sys.exit(1) 43 | return proc 44 | 45 | 46 | def configure_args(*args, **kwargs): 47 | cmd = [] 48 | for arg in args: 49 | switch = re.sub("_", "-", arg) 50 | cmd.append(f"--{switch}") 51 | for arg, value in kwargs.items(): 52 | switch = re.sub("_", "-", arg) 53 | cmd.append(f"--{switch}") 54 | if isinstance(value, list): 55 | cmd.extend(value) 56 | else: 57 | cmd.append(value) 58 | return cmd 59 | 60 | 61 | def run_litani(litani, subcommand, *args, check=True, **kwargs): 62 | cmd = [litani, subcommand] 63 | cmd.extend(configure_args(*args, **kwargs)) 64 | return run_cmd(cmd, check) 65 | 66 | 67 | def get_test_module(module_file): 68 | sys.path.insert(1, str(module_file.parent)) 69 | return importlib.import_module(str(module_file.stem)) 70 | 71 | 72 | def check_run(_, run_dir, mod): 73 | os.chdir(run_dir) 74 | with open(run_dir / "output" / "run.json") as handle: 75 | run = json.load(handle) 76 | print(json.dumps(run, indent=2)) 77 | if not mod.check_run(run): 78 | sys.exit(1) 79 | 80 | 81 | def add_jobs(litani, run_dir, mod): 82 | os.chdir(run_dir) 83 | jobs = mod.get_jobs() 84 | for job in jobs: 85 | run_litani( 86 | litani, "add-job", *job.get("args", []), **job.get("kwargs", {})) 87 | 88 | 89 | def transform_jobs(litani, run_dir, mod): 90 | os.chdir(run_dir) 91 | proc = subprocess.Popen( 92 | [litani, "transform-jobs"], stdin=subprocess.PIPE, 93 | stdout=subprocess.PIPE, text=True, bufsize=0) 94 | 95 | old_jobs = json.loads(proc.stdout.read()) 96 | new_jobs = mod.transform_jobs(old_jobs) 97 | 98 | print(json.dumps(new_jobs), file=proc.stdin) 99 | proc.stdin.flush() 100 | proc.stdin.close() 101 | 102 | proc.wait() 103 | if proc.returncode: 104 | logging.error("Return code: %d", proc.returncode) 105 | sys.exit(1) 106 | 107 | 108 | def get_jobs(litani, run_dir, mod): 109 | os.chdir(run_dir) 110 | args = mod.get_check_get_jobs_args() 111 | 112 | cmd = [litani, "get-jobs"] 113 | cmd.extend(configure_args(*args.get("args", []), **args.get("kwargs", {}))) 114 | jobs = subprocess.check_output(cmd) 115 | 116 | if not mod.check_get_jobs(jobs): 117 | sys.exit(1) 118 | 119 | for job in mod.get_post_check_jobs(): 120 | run_litani( 121 | litani, "add-job", *job.get("args", []), **job.get("kwargs", {})) 122 | 123 | 124 | def set_jobs(litani, run_dir, mod): 125 | os.chdir(run_dir) 126 | args = mod.get_set_jobs_args() 127 | run_litani( 128 | litani, "set-jobs", *args.get("args", []), **args.get("kwargs", {})) 129 | 130 | 131 | def run_build(litani, run_dir, mod): 132 | os.chdir(run_dir) 133 | args = mod.get_run_build_args() 134 | proc = run_litani( 135 | litani, "run-build", 136 | *args.get("args", []), check=False, **args.get("kwargs", {})) 137 | 138 | try: 139 | expected_rc = mod.get_run_build_return_code() 140 | if expected_rc != proc.returncode: 141 | logging.error( 142 | "Expected return code %d did not match actual " 143 | "return code %d", expected_rc, proc.returncode) 144 | sys.exit(1) 145 | except AttributeError: 146 | # test does not have a get_run_build_return_code method, so assume that 147 | # run-build is expected to succeed 148 | if proc.returncode: 149 | logging.error("Invocation failed") 150 | sys.exit(1) 151 | 152 | 153 | 154 | def init(litani, run_dir, mod): 155 | os.chdir(run_dir) 156 | args = mod.get_init_args() 157 | 158 | kwargs = args.get("kwargs", {}) 159 | kwargs.pop("output-directory", None) 160 | kwargs.pop("output-prefix", None) 161 | kwargs.pop("output-symlink", None) 162 | kwargs["output-directory"] = run_dir / "output" 163 | 164 | run_litani( 165 | litani, "init", *args.get("args", []), **kwargs) 166 | 167 | 168 | def add_global_context(args): 169 | os.environ["LITANI_E2E_LITANI_PATH"] = str(args.litani.resolve()) 170 | 171 | 172 | OPERATIONS = { 173 | "init": init, 174 | "add-jobs": add_jobs, 175 | "run-build": run_build, 176 | "check-run": check_run, 177 | "transform-jobs": transform_jobs, 178 | "get-jobs": get_jobs, 179 | "set-jobs": set_jobs, 180 | } 181 | 182 | 183 | def get_args(): 184 | pars = argparse.ArgumentParser(description=DESCRIPTION, epilog=EPILOG) 185 | for arg in [{ 186 | "flags": ["--test-file"], 187 | "required": True, 188 | "type": pathlib.Path, 189 | "help": "file under test/e2e/tests containing test definition", 190 | }, { 191 | "flags": ["--litani"], 192 | "required": True, 193 | "type": pathlib.Path, 194 | "help": "path to Litani under test", 195 | }, { 196 | "flags": ["--run-dir"], 197 | "required": True, 198 | "type": pathlib.Path, 199 | "help": "a fresh directory in which Litani will be run", 200 | }, { 201 | "flags": ["--operation"], 202 | "required": True, 203 | "choices": list(OPERATIONS.keys()) 204 | }]: 205 | flags = arg.pop("flags") 206 | pars.add_argument(*flags, **arg) 207 | return pars.parse_args() 208 | 209 | 210 | def main(): 211 | args = get_args() 212 | logging.basicConfig(format="run-tests: %(message)s") 213 | 214 | add_global_context(args) 215 | 216 | mod = get_test_module(args.test_file) 217 | 218 | OPERATIONS[args.operation](args.litani, args.run_dir, mod) 219 | 220 | 221 | if __name__ == "__main__": 222 | main() 223 | -------------------------------------------------------------------------------- /test/e2e/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/aws-build-accumulator/8002c240ef4f424039ed3cc32e076c0234d01768/test/e2e/tests/__init__.py -------------------------------------------------------------------------------- /test/e2e/tests/custom_stages.py: -------------------------------------------------------------------------------- 1 | NUM_STAGES = 4 2 | SLOW = False 3 | 4 | def get_stages(): 5 | stages = [] 6 | for i in range(1, NUM_STAGES + 1): 7 | stages.append("stage_" + str(i)) 8 | return stages 9 | 10 | def get_init_args(): 11 | return { 12 | "kwargs": { 13 | "project": "custom_stages", 14 | "stages": get_stages(), 15 | } 16 | } 17 | 18 | def get_jobs(): 19 | jobs = [] 20 | for i in range(1, NUM_STAGES + 1): 21 | job = { 22 | "kwargs": { 23 | "command": "sleep 2", 24 | "ci-stage": "stage_" + str(i), 25 | "description": str(i), 26 | "pipeline": "siesta", 27 | } 28 | } 29 | jobs.append(job) 30 | return jobs 31 | 32 | def get_run_build_args(): 33 | return {} 34 | 35 | def check_run(run): 36 | asserts = [] 37 | 38 | stages = get_stages() 39 | stages_run = run["stages"] 40 | 41 | # Assert that the expected stages are 42 | # equal to the stages stored in the run 43 | asserts.append(stages == stages_run) 44 | 45 | pipe_stages = run["pipelines"][0]["ci_stages"] 46 | for pipe_stage in pipe_stages: 47 | # Assert that any stage in the pipeline is one of the expected stages 48 | asserts.append(pipe_stage["name"] in stages) 49 | for job in pipe_stage["jobs"]: 50 | # Assert that the stage of a job corresponds to the pipeline stage 51 | asserts.append(job["wrapper_arguments"]["ci_stage"] == pipe_stage["name"]) 52 | # Assert that the description of a job corresponds to the expected one 53 | asserts.append("stage_" + job["wrapper_arguments"]["description"] == pipe_stage["name"]) 54 | 55 | # Check that the number of asserts is the expected one 56 | num_asserts = 3 * NUM_STAGES + 1 57 | 58 | return num_asserts == len(asserts) and all(asserts) 59 | -------------------------------------------------------------------------------- /test/e2e/tests/cwd.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "pwd", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | "cwd": "/", 31 | } 32 | }] 33 | 34 | 35 | def get_run_build_args(): 36 | return {} 37 | 38 | 39 | def check_run(run): 40 | pipe = run["pipelines"][0] 41 | job = pipe["ci_stages"][0]["jobs"][0] 42 | 43 | return job["stdout"] == ["/"] 44 | -------------------------------------------------------------------------------- /test/e2e/tests/dump_run.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | import json 17 | import os 18 | import pathlib 19 | import sys 20 | 21 | 22 | def get_init_args(): 23 | return { 24 | "kwargs": { 25 | "project": "foo", 26 | } 27 | } 28 | 29 | 30 | def get_jobs(): 31 | litani_path = pathlib.Path(os.environ["LITANI_E2E_LITANI_PATH"]) 32 | if not litani_path.exists(): 33 | raise UserWarning("Could not find litani executable") 34 | 35 | return [{ 36 | "kwargs": { 37 | "command": f"sleep 2", 38 | "description": "SLEEP", 39 | "outputs": "sleep-output", 40 | "ci-stage": "build", 41 | "pipeline": "foo", 42 | } 43 | }, { 44 | "kwargs": { 45 | "command": f"{litani_path.resolve()} dump-run", 46 | "description": "DUMP", 47 | "inputs": "sleep-output", 48 | "timeout": 20, 49 | "ci-stage": "build", 50 | "pipeline": "foo", 51 | } 52 | }, { 53 | "kwargs": { 54 | "command": "sleep 10", 55 | "description": "LONG", 56 | "ci-stage": "build", 57 | "pipeline": "foo", 58 | } 59 | }] 60 | 61 | 62 | def get_run_build_args(): 63 | return {} 64 | 65 | 66 | def perror(*args, **kwargs): 67 | kwargs = { 68 | **kwargs, "file": sys.stderr 69 | } 70 | print(*args, **kwargs) 71 | 72 | 73 | def check_run(run): 74 | pipe = run["pipelines"][0] 75 | jobs = pipe["ci_stages"][0]["jobs"] 76 | dump_job = [ 77 | j for j in jobs 78 | if j["wrapper_arguments"]["description"] == "DUMP" 79 | ] 80 | if not dump_job: 81 | perror("Dump job doesn't exist") 82 | return False 83 | 84 | run_dump = "\n".join(dump_job[0]["stdout"]) 85 | try: 86 | run_so_far = json.loads(run_dump) 87 | except json.decoder.JSONDecodeError: 88 | perror("JSON decode") 89 | perror(run_dump) 90 | return False 91 | 92 | pipe_so_far = run_so_far["pipelines"][0] 93 | jobs = pipe_so_far["ci_stages"][0]["jobs"] 94 | 95 | sleep_job = [ 96 | j for j in jobs 97 | if j["wrapper_arguments"]["description"] == "SLEEP" 98 | ] 99 | if not sleep_job: 100 | perror("sleep job doesn't exist") 101 | return False 102 | 103 | if not all(( 104 | sleep_job[0]["complete"], 105 | sleep_job[0]["duration"] > 1, 106 | )): 107 | perror("sleep job format") 108 | return False 109 | 110 | long_job = [ 111 | j for j in jobs 112 | if j["wrapper_arguments"]["description"] == "LONG" 113 | ] 114 | if not long_job: 115 | perror("long job doesn't exist") 116 | return False 117 | 118 | if long_job[0]["complete"]: 119 | perror("long job already complete before dumping run") 120 | return False 121 | 122 | return True 123 | -------------------------------------------------------------------------------- /test/e2e/tests/exit_rc.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "false", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | "ignore_returns": "1", 31 | } 32 | }] 33 | 34 | 35 | def get_run_build_args(): 36 | return {} 37 | 38 | 39 | def get_run_build_return_code(): 40 | return 0 41 | 42 | 43 | def check_run(run): 44 | return True 45 | -------------------------------------------------------------------------------- /test/e2e/tests/exit_rc_fail_on_pipeline_failure.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "false", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | "ignore_returns": "1", 31 | } 32 | }] 33 | 34 | 35 | def get_run_build_args(): 36 | return { 37 | "args": [ 38 | "fail-on-pipeline-failure", 39 | ] 40 | } 41 | 42 | 43 | def get_run_build_return_code(): 44 | return 10 45 | 46 | 47 | def check_run(run): 48 | return True 49 | -------------------------------------------------------------------------------- /test/e2e/tests/get_jobs_multiple_jobs.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import json 15 | import logging 16 | import sys 17 | 18 | 19 | SLOW = False 20 | 21 | 22 | def get_init_args(): 23 | return { 24 | "kwargs": { 25 | "project": "foo", 26 | } 27 | } 28 | 29 | 30 | def get_jobs(): 31 | return [{ 32 | "kwargs": { 33 | "command": "echo foo", 34 | "ci-stage": "build", 35 | "pipeline": "foo", 36 | } 37 | }, { 38 | "kwargs": { 39 | "command": "echo bar", 40 | "ci-stage": "build", 41 | "pipeline": "foo", 42 | } 43 | }, { 44 | "kwargs": { 45 | "command": "echo baz", 46 | "ci-stage": "build", 47 | "pipeline": "foo" 48 | } 49 | }] 50 | 51 | 52 | def get_check_get_jobs_args(): 53 | return {} 54 | 55 | 56 | 57 | def check_get_jobs(jobs_json): 58 | jobs = json.loads(jobs_json) 59 | if len(jobs) != 3: 60 | logging.error("Expected to get 3 job, got %s", len(jobs)) 61 | return False 62 | 63 | expected_commands = { 64 | "echo bar", 65 | "echo baz", 66 | "echo foo", 67 | } 68 | actual_commands = set([j["command"] for j in jobs]) 69 | if actual_commands != expected_commands: 70 | logging.error("Expected job commands to be %s, actual %s", 71 | expected_commands, 72 | actual_commands) 73 | return False 74 | return True 75 | 76 | 77 | def get_post_check_jobs(): 78 | return [] 79 | 80 | 81 | def get_run_build_args(): 82 | return {} 83 | 84 | 85 | def check_run(run): 86 | return len(run["pipelines"][0]["ci_stages"][0]["jobs"]) == 3 87 | -------------------------------------------------------------------------------- /test/e2e/tests/get_jobs_no_jobs.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import json 15 | import logging 16 | import sys 17 | 18 | 19 | SLOW = False 20 | PHONY_ADD_JOBS = True 21 | 22 | 23 | def get_init_args(): 24 | return { 25 | "kwargs": { 26 | "project": "foo", 27 | } 28 | } 29 | 30 | 31 | def get_jobs(): 32 | return [] 33 | 34 | 35 | def get_check_get_jobs_args(): 36 | return {} 37 | 38 | 39 | def check_get_jobs(jobs_json): 40 | jobs = json.loads(jobs_json) 41 | if jobs != []: 42 | logging.error("Not expecting to get any jobs, got %s", jobs) 43 | return False 44 | return True 45 | 46 | 47 | def get_post_check_jobs(): 48 | return [{ 49 | "kwargs": { 50 | "command": "echo foo", 51 | "ci-stage": "build", 52 | "pipeline": "foo", 53 | } 54 | }] 55 | 56 | 57 | def get_run_build_args(): 58 | return {} 59 | 60 | 61 | def check_run(run): 62 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 63 | return job["stdout"][0].strip() == "foo" 64 | -------------------------------------------------------------------------------- /test/e2e/tests/get_jobs_single_job.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import json 15 | import logging 16 | 17 | 18 | SLOW = False 19 | 20 | EXPECTED_JOB = { 21 | "command": "echo foo", 22 | "ci_stage": "build", 23 | "pipeline_name": "foo", 24 | } 25 | 26 | 27 | def get_init_args(): 28 | return { 29 | "kwargs": { 30 | "project": "foo", 31 | } 32 | } 33 | 34 | 35 | def get_jobs(): 36 | return [{"kwargs": EXPECTED_JOB}] 37 | 38 | 39 | def get_check_get_jobs_args(): 40 | return {} 41 | 42 | 43 | def check_get_jobs(jobs_json): 44 | jobs = json.loads(jobs_json) 45 | print(json.dumps(jobs, indent=2)) 46 | if len(jobs) != 1: 47 | logging.error("Expected to get 1 job, got %s", len(jobs)) 48 | return False 49 | 50 | job = jobs[0] 51 | for key in EXPECTED_JOB.keys(): 52 | if job.get(key) != EXPECTED_JOB[key]: 53 | logging.error( 54 | "Expected job to have key-value pair '%s': '%s'", 55 | key, EXPECTED_JOB[key]) 56 | return False 57 | 58 | removed_keys = [ 59 | "job_id", 60 | "status_file", 61 | "subcommand", 62 | ] 63 | for key in removed_keys: 64 | if key in job: 65 | logging.error("Expected key %s to be removed from job output", key) 66 | return False 67 | return True 68 | 69 | 70 | def get_post_check_jobs(): 71 | return [] 72 | 73 | 74 | def get_run_build_args(): 75 | return {} 76 | 77 | 78 | def check_run(run): 79 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 80 | return job["stdout"][0].strip() == "foo" 81 | -------------------------------------------------------------------------------- /test/e2e/tests/graph_line_break.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "echo '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | } 31 | }] 32 | 33 | 34 | def get_run_build_args(): 35 | return {} 36 | 37 | 38 | def check_run(run): 39 | return True 40 | -------------------------------------------------------------------------------- /test/e2e/tests/html_node.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": 'echo HTML chars: \& \\\" \< \> >/dev/null', 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | } 31 | }, { 32 | "kwargs": { 33 | "command": 'echo HTML chars: \& \\\" \< \> >/dev/null', 34 | "description": '" && & < <>> > "', 35 | "ci-stage": "build", 36 | "pipeline": "foo", 37 | } 38 | }] 39 | 40 | 41 | def get_run_build_args(): 42 | return {} 43 | 44 | 45 | def check_run(run): 46 | return True 47 | -------------------------------------------------------------------------------- /test/e2e/tests/job_id_env.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "echo $$LITANI_JOB_ID", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | } 31 | }] 32 | 33 | 34 | def get_run_build_args(): 35 | return {} 36 | 37 | 38 | def check_run(run): 39 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 40 | return job["wrapper_arguments"]["job_id"] == job["stdout"][0].strip() 41 | -------------------------------------------------------------------------------- /test/e2e/tests/multiproc_dump_run.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | import json 17 | import os 18 | import pathlib 19 | import sys 20 | 21 | 22 | def get_init_args(): 23 | return { 24 | "kwargs": { 25 | "project": "foo", 26 | } 27 | } 28 | 29 | 30 | def get_jobs(): 31 | litani_path = pathlib.Path(os.environ["LITANI_E2E_LITANI_PATH"]) 32 | if not litani_path.exists(): 33 | raise UserWarning("Could not find litani executable") 34 | 35 | ret = [] 36 | for i in range(50): 37 | ret.extend([{ 38 | "kwargs": { 39 | "command": f"sleep 1", 40 | "description": f"sleep-{i}", 41 | "outputs": f"output-{i}", 42 | "ci-stage": "build", 43 | "pipeline": "foo", 44 | } 45 | }, { 46 | "kwargs": { 47 | "command": 48 | f"{litani_path} dump-run " 49 | """| jq -e '.. | .jobs? | .[]? """ 50 | f"""| .complete or .wrapper_arguments.job_id != "sleep-{i}"'""", 51 | "description": f"test-{i}", 52 | "inputs": f"output-{i}", 53 | "ci-stage": "build", 54 | "pipeline": "foo", 55 | } 56 | }]) 57 | return ret 58 | 59 | 60 | def get_run_build_args(): 61 | return {} 62 | 63 | 64 | def check_run(run): 65 | pipe = run["pipelines"][0] 66 | jobs = pipe["ci_stages"][0]["jobs"] 67 | for job in jobs: 68 | if not job["outcome"] == "success": 69 | return False 70 | return True 71 | -------------------------------------------------------------------------------- /test/e2e/tests/no_pool_serialize.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "sleep 2", 28 | "description": "first", 29 | "ci-stage": "build", 30 | "pipeline": "foo", 31 | } 32 | }, { 33 | "kwargs": { 34 | "command": "sleep 2", 35 | "description": "second", 36 | "ci-stage": "build", 37 | "pipeline": "foo", 38 | }, 39 | }, { 40 | "kwargs": { 41 | "command": "sleep 2", 42 | "description": "third", 43 | "ci-stage": "build", 44 | "pipeline": "foo", 45 | }, 46 | }] 47 | 48 | 49 | def get_run_build_args(): 50 | return {} 51 | 52 | 53 | def check_run(run): 54 | jobs = run["pipelines"][0]["ci_stages"][0]["jobs"] 55 | 56 | first = [j for j in jobs if j["wrapper_arguments"]["description"] == "first"][0] 57 | second = [j for j in jobs if j["wrapper_arguments"]["description"] == "second"][0] 58 | third = [j for j in jobs if j["wrapper_arguments"]["description"] == "third"][0] 59 | 60 | return all(( 61 | first["end_time"] > second["start_time"], 62 | second["end_time"] > third["start_time"], 63 | third["end_time"] > first["start_time"], 64 | )) 65 | -------------------------------------------------------------------------------- /test/e2e/tests/no_pool_serialize_graph.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "sleep 2", 28 | "description": "first", 29 | "ci-stage": "build", 30 | "pipeline": "foo", 31 | } 32 | }, { 33 | "kwargs": { 34 | "command": "sleep 2", 35 | "description": "second", 36 | "ci-stage": "build", 37 | "pipeline": "foo", 38 | }, 39 | }, { 40 | "kwargs": { 41 | "command": "sleep 2", 42 | "description": "third", 43 | "ci-stage": "build", 44 | "pipeline": "foo", 45 | }, 46 | }] 47 | 48 | 49 | def get_run_build_args(): 50 | return {} 51 | 52 | 53 | def check_run(run): 54 | return run["parallelism"]["max_parallelism"] == 3 55 | -------------------------------------------------------------------------------- /test/e2e/tests/no_timed_out.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "sleep 1", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | "timeout": "30", 31 | } 32 | }] 33 | 34 | 35 | def get_run_build_args(): 36 | return {} 37 | 38 | 39 | def check_run(run): 40 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 41 | return not job["timeout_reached"] 42 | -------------------------------------------------------------------------------- /test/e2e/tests/no_timed_out_timeout_ignored.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "args": [ 27 | "timeout-ignore", 28 | ], 29 | "kwargs": { 30 | "command": "sleep 1", 31 | "ci-stage": "build", 32 | "pipeline": "foo", 33 | "timeout": "10", 34 | } 35 | }] 36 | 37 | 38 | def get_run_build_args(): 39 | return {} 40 | 41 | 42 | def check_run(run): 43 | pipe = run["pipelines"][0] 44 | job = pipe["ci_stages"][0]["jobs"][0] 45 | 46 | return all(( 47 | not job["timeout_reached"], 48 | pipe["status"] == "success", 49 | )) 50 | -------------------------------------------------------------------------------- /test/e2e/tests/no_timed_out_timeout_ok.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "args": [ 27 | "timeout-ok", 28 | ], 29 | "kwargs": { 30 | "command": "sleep 1", 31 | "ci-stage": "build", 32 | "pipeline": "foo", 33 | "timeout": "10", 34 | } 35 | }] 36 | 37 | 38 | def get_run_build_args(): 39 | return {} 40 | 41 | 42 | def check_run(run): 43 | pipe = run["pipelines"][0] 44 | job = pipe["ci_stages"][0]["jobs"][0] 45 | 46 | return all(( 47 | not job["timeout_reached"], 48 | pipe["status"] == "success", 49 | )) 50 | -------------------------------------------------------------------------------- /test/e2e/tests/pipeline_order.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | EXPECTED_PIPELINE_NAMES = ["zeta", "0", "00", "01", "1", "Beta", "alpha", 16 | "beta", "beta1", "beta2", "betaa", "delta", "epsilon", "gamma"] 17 | 18 | 19 | def get_init_args(): 20 | return { 21 | "kwargs": { 22 | "project": "sorted_pipeline_names", 23 | } 24 | } 25 | 26 | 27 | def get_jobs(): 28 | jobs = [] 29 | jobs.append({ 30 | "kwargs": { 31 | "command": "/usr/bin/false", 32 | "ci-stage": "build", 33 | "pipeline": EXPECTED_PIPELINE_NAMES[0], 34 | } 35 | }) 36 | for pipeline in EXPECTED_PIPELINE_NAMES[1:]: 37 | job = { 38 | "kwargs": { 39 | "command": "/usr/bin/true", 40 | "ci-stage": "build", 41 | "pipeline": pipeline, 42 | } 43 | } 44 | jobs.append(job) 45 | return jobs 46 | 47 | 48 | def get_run_build_args(): 49 | return {} 50 | 51 | 52 | def check_run(run): 53 | return [p["name"] for p in run["pipelines"]] == EXPECTED_PIPELINE_NAMES 54 | -------------------------------------------------------------------------------- /test/e2e/tests/pool_serialize.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | "pools": [ "foo-pool:1" ] 21 | } 22 | } 23 | 24 | 25 | def get_jobs(): 26 | return [{ 27 | "kwargs": { 28 | "command": "sleep 2", 29 | "description": "first", 30 | "ci-stage": "build", 31 | "pipeline": "foo", 32 | "pool": "foo-pool" 33 | } 34 | }, { 35 | "kwargs": { 36 | "command": "sleep 2", 37 | "description": "second", 38 | "ci-stage": "build", 39 | "pipeline": "foo", 40 | "pool": "foo-pool" 41 | }, 42 | }, { 43 | "kwargs": { 44 | "command": "sleep 2", 45 | "description": "third", 46 | "ci-stage": "build", 47 | "pipeline": "foo", 48 | "pool": "foo-pool" 49 | }, 50 | }] 51 | 52 | 53 | def get_run_build_args(): 54 | return {} 55 | 56 | 57 | def check_run(run): 58 | jobs = run["pipelines"][0]["ci_stages"][0]["jobs"] 59 | 60 | first = [j for j in jobs if j["wrapper_arguments"]["description"] == "first"][0] 61 | second = [j for j in jobs if j["wrapper_arguments"]["description"] == "second"][0] 62 | third = [j for j in jobs if j["wrapper_arguments"]["description"] == "third"][0] 63 | 64 | return all(( 65 | any(( 66 | first["start_time"] < second["start_time"], 67 | not (first["end_time"] <= second["start_time"]), 68 | )), 69 | any(( 70 | second["start_time"] < third["start_time"], 71 | not (second["end_time"] <= third["start_time"]), 72 | )), 73 | any(( 74 | third["start_time"] < first["start_time"], 75 | not (third["end_time"] <= first["start_time"]), 76 | )), 77 | )) 78 | -------------------------------------------------------------------------------- /test/e2e/tests/pool_serialize_graph.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | "pools": [ "foo-pool:1" ] 21 | } 22 | } 23 | 24 | 25 | def get_jobs(): 26 | return [{ 27 | "kwargs": { 28 | "command": "sleep 2", 29 | "description": "first", 30 | "ci-stage": "build", 31 | "pipeline": "foo", 32 | "pool": "foo-pool" 33 | } 34 | }, { 35 | "kwargs": { 36 | "command": "sleep 2", 37 | "description": "second", 38 | "ci-stage": "build", 39 | "pipeline": "foo", 40 | "pool": "foo-pool" 41 | }, 42 | }, { 43 | "kwargs": { 44 | "command": "sleep 2", 45 | "description": "third", 46 | "ci-stage": "build", 47 | "pipeline": "foo", 48 | "pool": "foo-pool" 49 | }, 50 | }] 51 | 52 | 53 | def get_run_build_args(): 54 | return {} 55 | 56 | 57 | def check_run(run): 58 | return run["parallelism"]["max_parallelism"] == 1 59 | -------------------------------------------------------------------------------- /test/e2e/tests/set_jobs_no_jobs.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import json 15 | import logging 16 | import sys 17 | 18 | 19 | SLOW = False 20 | PHONY_ADD_JOBS = True 21 | 22 | 23 | def get_init_args(): 24 | return { 25 | "kwargs": { 26 | "project": "foo", 27 | } 28 | } 29 | 30 | 31 | def get_jobs(): 32 | return [] 33 | 34 | 35 | def get_set_jobs_args(): 36 | jobs = [{ 37 | "command": "echo foo", 38 | "pipeline_name": "foo", 39 | "ci_stage": "build", 40 | }] 41 | return { 42 | "kwargs": { 43 | "from_string": json.dumps(jobs) 44 | } 45 | } 46 | 47 | 48 | def get_run_build_args(): 49 | return {} 50 | 51 | 52 | def check_run(run): 53 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 54 | return job["stdout"][0].strip() == "foo" 55 | -------------------------------------------------------------------------------- /test/e2e/tests/set_jobs_overwrite.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import json 15 | import logging 16 | import sys 17 | 18 | 19 | SLOW = False 20 | 21 | 22 | def get_init_args(): 23 | return { 24 | "kwargs": { 25 | "project": "foo", 26 | } 27 | } 28 | 29 | 30 | def get_jobs(): 31 | return [{ 32 | "kwargs": { 33 | "command": "echo foo", 34 | "ci-stage": "build", 35 | "pipeline": "foo", 36 | } 37 | }] 38 | 39 | 40 | def get_set_jobs_args(): 41 | jobs = [{ 42 | "command": "echo bar", 43 | "pipeline_name": "foo", 44 | "ci_stage": "build", 45 | }] 46 | return { 47 | "kwargs": { 48 | "from_string": json.dumps(jobs) 49 | } 50 | } 51 | 52 | 53 | def get_run_build_args(): 54 | return {} 55 | 56 | 57 | def check_run(run): 58 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 59 | return job["stdout"][0].strip() == "bar" 60 | -------------------------------------------------------------------------------- /test/e2e/tests/set_multiple_jobs.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import json 15 | import logging 16 | import sys 17 | 18 | 19 | SLOW = False 20 | PHONY_ADD_JOBS = True 21 | 22 | 23 | def get_init_args(): 24 | return { 25 | "kwargs": { 26 | "project": "foo", 27 | } 28 | } 29 | 30 | 31 | def get_jobs(): 32 | return [] 33 | 34 | 35 | def get_set_jobs_args(): 36 | jobs = [{ 37 | "command": "echo foo", 38 | "pipeline_name": "foo", 39 | "ci_stage": "build", 40 | }, { 41 | "command": "echo baz", 42 | "pipeline_name": "foo", 43 | "ci_stage": "build", 44 | }, { 45 | "command": "echo bar", 46 | "pipeline_name": "foo", 47 | "ci_stage": "build", 48 | }] 49 | return { 50 | "kwargs": { 51 | "from_string": json.dumps(jobs) 52 | } 53 | } 54 | 55 | 56 | def get_run_build_args(): 57 | return {} 58 | 59 | 60 | def check_run(run): 61 | jobs = run["pipelines"][0]["ci_stages"][0]["jobs"] 62 | 63 | expected_outputs = ["foo", "bar", "baz"] 64 | for job in jobs: 65 | expected_outputs.remove(job["stdout"][0].strip()) 66 | 67 | return all(( 68 | len(jobs) == 3, 69 | expected_outputs == [], 70 | )) 71 | -------------------------------------------------------------------------------- /test/e2e/tests/single_pool.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | "pools": [ "foo-pool:1" ] 21 | } 22 | } 23 | 24 | 25 | def get_jobs(): 26 | return [{ 27 | "kwargs": { 28 | "command": "true", 29 | "ci-stage": "build", 30 | "pipeline": "foo" 31 | } 32 | }] 33 | 34 | 35 | def get_run_build_args(): 36 | return {} 37 | 38 | 39 | def check_run(run): 40 | return run.get("pools") == { "foo-pool": 1 } 41 | -------------------------------------------------------------------------------- /test/e2e/tests/timed_out.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "sleep 10", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | "timeout": "3", 31 | } 32 | }] 33 | 34 | 35 | def get_run_build_args(): 36 | return {} 37 | 38 | 39 | def check_run(run): 40 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 41 | return job["timeout_reached"] 42 | -------------------------------------------------------------------------------- /test/e2e/tests/timed_out_subprocess.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": """\ 28 | /usr/bin/python3 -c 'import subprocess; \ 29 | subprocess.run(["sleep", "10"])' \ 30 | """, 31 | "ci-stage": "build", 32 | "pipeline": "foo", 33 | "timeout": "3", 34 | } 35 | }] 36 | 37 | 38 | def get_run_build_args(): 39 | return {} 40 | 41 | 42 | def check_run(run): 43 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 44 | return all((job["timeout_reached"], job["duration"] < 5)) 45 | -------------------------------------------------------------------------------- /test/e2e/tests/timed_out_subprocess_multi_shell.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": """\ 28 | /usr/bin/python3 -c 'import subprocess; \ 29 | subprocess.run(["sleep", "10"])'; /usr/bin/true \ 30 | """, 31 | "ci-stage": "build", 32 | "pipeline": "foo", 33 | "timeout": "3", 34 | } 35 | }] 36 | 37 | 38 | def get_run_build_args(): 39 | return {} 40 | 41 | 42 | def check_run(run): 43 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 44 | return all((job["timeout_reached"], job["duration"] < 5)) 45 | -------------------------------------------------------------------------------- /test/e2e/tests/timed_out_subprocess_shell.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": """\ 28 | python3 -c \"import subprocess; \ 29 | subprocess.run('sleep 10', shell=True)\" \ 30 | """, 31 | "ci-stage": "build", 32 | "pipeline": "foo", 33 | "timeout": "3", 34 | } 35 | }] 36 | 37 | 38 | def get_run_build_args(): 39 | return {} 40 | 41 | 42 | def check_run(run): 43 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 44 | return all((job["timeout_reached"], job["duration"] < 5)) 45 | -------------------------------------------------------------------------------- /test/e2e/tests/timed_out_timeout_ignored.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "args": [ 27 | "timeout-ignore", 28 | ], 29 | "kwargs": { 30 | "command": "sleep 10", 31 | "ci-stage": "build", 32 | "pipeline": "foo", 33 | "timeout": "3", 34 | } 35 | }] 36 | 37 | 38 | def get_run_build_args(): 39 | return {} 40 | 41 | 42 | def check_run(run): 43 | pipe = run["pipelines"][0] 44 | job = pipe["ci_stages"][0]["jobs"][0] 45 | 46 | return all(( 47 | job["timeout_reached"], 48 | job["outcome"] == "fail_ignored", 49 | pipe["status"] == "fail", 50 | )) 51 | -------------------------------------------------------------------------------- /test/e2e/tests/timed_out_timeout_ok.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = True 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "args": [ 27 | "timeout-ok", 28 | ], 29 | "kwargs": { 30 | "command": "sleep 10", 31 | "ci-stage": "build", 32 | "pipeline": "foo", 33 | "timeout": "3", 34 | } 35 | }] 36 | 37 | 38 | def get_run_build_args(): 39 | return {} 40 | 41 | 42 | def check_run(run): 43 | pipe = run["pipelines"][0] 44 | job = pipe["ci_stages"][0]["jobs"][0] 45 | 46 | return all(( 47 | job["timeout_reached"], 48 | pipe["status"] == "success", 49 | )) 50 | -------------------------------------------------------------------------------- /test/e2e/tests/transform_delete_job.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "echo foo", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | } 31 | }, { 32 | "kwargs": { 33 | "command": "echo bar", 34 | "ci-stage": "build", 35 | "pipeline": "foo", 36 | } 37 | }] 38 | 39 | 40 | def transform_jobs(jobs): 41 | new_jobs = [] 42 | for job in jobs: 43 | if job["command"] != "echo bar": 44 | new_jobs.append(job) 45 | return new_jobs 46 | 47 | 48 | def get_run_build_args(): 49 | return {} 50 | 51 | 52 | def check_run(run): 53 | jobs = run["pipelines"][0]["ci_stages"][0]["jobs"] 54 | return all((len(jobs) == 1, jobs[0]["stdout"][0].strip() == "foo")) 55 | -------------------------------------------------------------------------------- /test/e2e/tests/transform_modify_job.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "echo foo", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | } 31 | }] 32 | 33 | 34 | def transform_jobs(jobs): 35 | new_jobs = list(jobs) 36 | new_jobs[0]["command"] = "echo bar" 37 | return new_jobs 38 | 39 | 40 | def get_run_build_args(): 41 | return {} 42 | 43 | 44 | def check_run(run): 45 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 46 | return job["stdout"][0].strip() == "bar" 47 | -------------------------------------------------------------------------------- /test/e2e/tests/transform_no_change_job.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "echo foo", 28 | "ci-stage": "build", 29 | "pipeline": "foo", 30 | } 31 | }] 32 | 33 | 34 | def transform_jobs(jobs): 35 | return list(jobs) 36 | 37 | 38 | def get_run_build_args(): 39 | return {} 40 | 41 | 42 | def check_run(run): 43 | job = run["pipelines"][0]["ci_stages"][0]["jobs"][0] 44 | return job["stdout"][0].strip() == "foo" 45 | -------------------------------------------------------------------------------- /test/e2e/tests/zero_pool.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | SLOW = False 15 | 16 | def get_init_args(): 17 | return { 18 | "kwargs": { 19 | "project": "foo", 20 | } 21 | } 22 | 23 | 24 | def get_jobs(): 25 | return [{ 26 | "kwargs": { 27 | "command": "true", 28 | "ci-stage": "build", 29 | "pipeline": "foo" 30 | } 31 | }] 32 | 33 | 34 | def get_run_build_args(): 35 | return {} 36 | 37 | 38 | def check_run(run): 39 | return run.get("pools") == {} 40 | -------------------------------------------------------------------------------- /test/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/aws-build-accumulator/8002c240ef4f424039ed3cc32e076c0234d01768/test/unit/__init__.py -------------------------------------------------------------------------------- /test/unit/lockable_directory.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | import pathlib 16 | import tempfile 17 | import unittest 18 | import unittest.mock 19 | 20 | import lib.litani 21 | 22 | 23 | 24 | class TestLockableDirectory(unittest.TestCase): 25 | def setUp(self): 26 | self.temp_dir = tempfile.TemporaryDirectory(prefix="litani-test") 27 | self.temp_dir_p = pathlib.Path(self.temp_dir.name) 28 | 29 | 30 | def tearDown(self): 31 | self.temp_dir.cleanup() 32 | 33 | 34 | def test_cannot_initially_acquire(self): 35 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 36 | self.assertFalse(lock_dir.acquire()) 37 | 38 | 39 | def test_multiproc_cannot_initially_acquire(self): 40 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 41 | self.assertFalse(lock_dir.acquire()) 42 | lock_dir_2 = lib.litani.LockableDirectory(self.temp_dir_p) 43 | self.assertFalse(lock_dir.acquire()) 44 | 45 | 46 | def test_can_acquire_after_release(self): 47 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 48 | lock_dir.release() 49 | self.assertTrue(lock_dir.acquire()) 50 | lock_dir.release() 51 | self.assertTrue(lock_dir.acquire()) 52 | 53 | 54 | def test_multiproc_can_acquire_after_release(self): 55 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 56 | lock_dir.release() 57 | 58 | lock_dir_2 = lib.litani.LockableDirectory(self.temp_dir_p) 59 | self.assertTrue(lock_dir_2.acquire()) 60 | 61 | 62 | def test_no_double_acquire(self): 63 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 64 | lock_dir.release() 65 | self.assertTrue(lock_dir.acquire()) 66 | self.assertFalse(lock_dir.acquire()) 67 | 68 | 69 | def test_multiproc_no_double_acquire(self): 70 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 71 | lock_dir.release() 72 | lock_dir_2 = lib.litani.LockableDirectory(self.temp_dir_p) 73 | 74 | self.assertTrue(lock_dir.acquire()) 75 | self.assertFalse(lock_dir_2.acquire()) 76 | 77 | 78 | def test_repeat_no_double_acquire(self): 79 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 80 | lock_dir.release() 81 | self.assertTrue(lock_dir.acquire()) 82 | self.assertFalse(lock_dir.acquire()) 83 | 84 | lock_dir.release() 85 | 86 | self.assertTrue(lock_dir.acquire()) 87 | self.assertFalse(lock_dir.acquire()) 88 | 89 | 90 | def test_try_acquire(self): 91 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 92 | lock_dir.release() 93 | with lock_dir.try_acquire(): 94 | pass # No exception 95 | 96 | 97 | def test_no_double_try_acquire(self): 98 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 99 | lock_dir.release() 100 | with lock_dir.try_acquire(): 101 | with self.assertRaises(lib.litani.AcquisitionFailed): 102 | with lock_dir.try_acquire(): 103 | pass 104 | 105 | 106 | def test_multiproc_no_double_try_acquire(self): 107 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 108 | lock_dir.release() 109 | lock_dir_2 = lib.litani.LockableDirectory(self.temp_dir_p) 110 | 111 | with lock_dir.try_acquire(): 112 | with self.assertRaises(lib.litani.AcquisitionFailed): 113 | with lock_dir_2.try_acquire(): 114 | pass 115 | 116 | 117 | def test_no_try_acquire_acquire(self): 118 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 119 | lock_dir.release() 120 | with lock_dir.try_acquire(): 121 | self.assertFalse(lock_dir.acquire()) 122 | 123 | 124 | def test_multiproc_no_try_acquire_acquire(self): 125 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 126 | lock_dir.release() 127 | lock_dir_2 = lib.litani.LockableDirectory(self.temp_dir_p) 128 | 129 | with lock_dir.try_acquire(): 130 | self.assertFalse(lock_dir_2.acquire()) 131 | 132 | 133 | def test_no_acquire_try_acquire(self): 134 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 135 | with self.assertRaises(lib.litani.AcquisitionFailed): 136 | with lock_dir.try_acquire(): 137 | pass 138 | 139 | 140 | def test_multiproc_no_acquire_try_acquire(self): 141 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 142 | lock_dir_2 = lib.litani.LockableDirectory(self.temp_dir_p) 143 | with self.assertRaises(lib.litani.AcquisitionFailed): 144 | with lock_dir_2.try_acquire(): 145 | pass 146 | 147 | 148 | def test_context_manager_releases(self): 149 | lock_dir = lib.litani.LockableDirectory(self.temp_dir_p) 150 | lock_dir.release() 151 | with lock_dir.try_acquire(): 152 | pass 153 | self.assertTrue(lock_dir.acquire()) 154 | -------------------------------------------------------------------------------- /test/unit/output_artifact.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | import shutil 16 | import pathlib 17 | import unittest 18 | import unittest.mock 19 | 20 | import lib.output_artifact 21 | 22 | 23 | 24 | class TestOutputArtifact(unittest.TestCase): 25 | def setUp(self): 26 | shutil.copy = unittest.mock.Mock(side_effect=FileNotFoundError("")) 27 | self.artifacts_dir = pathlib.Path() 28 | 29 | 30 | def test_no_flag(self): 31 | job_args = {} 32 | copier = lib.output_artifact.Copier(self.artifacts_dir, job_args) 33 | 34 | with self.assertRaises(lib.output_artifact.MissingOutput): 35 | copier.copy_output_artifact("foo") 36 | 37 | 38 | def test_null_flag(self): 39 | job_args = { 40 | "phony_outputs": None, 41 | } 42 | copier = lib.output_artifact.Copier(self.artifacts_dir, job_args) 43 | 44 | with self.assertRaises(lib.output_artifact.MissingOutput): 45 | copier.copy_output_artifact("foo") 46 | 47 | 48 | def test_all_phony(self): 49 | job_args = { 50 | "phony_outputs": [], 51 | } 52 | copier = lib.output_artifact.Copier(self.artifacts_dir, job_args) 53 | copier.copy_output_artifact("foo") 54 | 55 | 56 | def test_different_phony(self): 57 | job_args = { 58 | "phony_outputs": ["bar"], 59 | } 60 | copier = lib.output_artifact.Copier(self.artifacts_dir, job_args) 61 | 62 | with self.assertRaises(lib.output_artifact.MissingOutput): 63 | copier.copy_output_artifact("foo") 64 | 65 | 66 | def test_file_phony(self): 67 | job_args = { 68 | "phony_outputs": ["foo"], 69 | } 70 | copier = lib.output_artifact.Copier(self.artifacts_dir, job_args) 71 | copier.copy_output_artifact("foo") 72 | -------------------------------------------------------------------------------- /test/unit/run_consistency.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | import unittest 16 | 17 | import lib.run_printer 18 | 19 | 20 | 21 | class TestRunConsistency(unittest.TestCase): 22 | def setUp(self): 23 | self.run = { 24 | "pipelines": [{ 25 | "ci_stages": [{ 26 | "jobs": [] 27 | }] 28 | }] 29 | } 30 | 31 | 32 | def add_jobs(self, *jobs): 33 | self.run["pipelines"][0]["ci_stages"][0]["jobs"].extend(jobs) 34 | 35 | 36 | def test_no_id(self): 37 | self.add_jobs({ 38 | "complete": True, 39 | "wrapper_arguments": { 40 | "job_id": "job 1", 41 | "outputs": None, 42 | "inputs": None, 43 | }}, { 44 | "complete": True, 45 | "wrapper_arguments": { 46 | "job_id": "job 2", 47 | "outputs": None, 48 | "inputs": None, 49 | }}) 50 | 51 | with self.assertRaises(lib.run_printer.InconsistentRunError): 52 | lib.run_printer.run_consistent_to_job(self.run, "job 3") 53 | 54 | 55 | def test_found_id(self): 56 | self.add_jobs({ 57 | "complete": False, 58 | "wrapper_arguments": { 59 | "job_id": "job 1", 60 | "outputs": None, 61 | "inputs": None, 62 | }}) 63 | 64 | self.assertTrue( 65 | lib.run_printer.run_consistent_to_job(self.run, "job 1")) 66 | 67 | 68 | def test_reverse_dep_incomplete(self): 69 | self.add_jobs({ 70 | "complete": False, 71 | "wrapper_arguments": { 72 | "job_id": "job 1", 73 | "outputs": ["foo"], 74 | "inputs": None, 75 | }}, { 76 | "complete": False, 77 | "wrapper_arguments": { 78 | "job_id": "job 2", 79 | "outputs": None, 80 | "inputs": ["foo"], 81 | }}) 82 | 83 | with self.assertRaises(lib.run_printer.InconsistentRunError): 84 | lib.run_printer.run_consistent_to_job(self.run, "job 2") 85 | 86 | 87 | def test_reverse_dep_complete(self): 88 | self.add_jobs({ 89 | "complete": True, 90 | "wrapper_arguments": { 91 | "job_id": "job 1", 92 | "outputs": ["foo"], 93 | "inputs": None, 94 | }}, { 95 | "complete": False, 96 | "wrapper_arguments": { 97 | "job_id": "job 2", 98 | "outputs": None, 99 | "inputs": ["foo"], 100 | }}) 101 | 102 | self.assertTrue( 103 | lib.run_printer.run_consistent_to_job(self.run, "job 2")) 104 | 105 | 106 | def test_unrelated_job_ok(self): 107 | self.add_jobs({ 108 | "complete": True, 109 | "wrapper_arguments": { 110 | "job_id": "job 1", 111 | "outputs": ["foo"], 112 | "inputs": None, 113 | }}, { 114 | "complete": False, 115 | "wrapper_arguments": { 116 | "job_id": "job 2", 117 | "outputs": None, 118 | "inputs": ["foo"], 119 | }}, { 120 | "complete": False, 121 | "wrapper_arguments": { 122 | "job_id": "job 3", 123 | "outputs": ["bar"], 124 | "inputs": None, 125 | }}) 126 | 127 | self.assertTrue( 128 | lib.run_printer.run_consistent_to_job(self.run, "job 2")) 129 | 130 | 131 | def test_both_deps_complete(self): 132 | self.add_jobs({ 133 | "complete": True, 134 | "wrapper_arguments": { 135 | "job_id": "job 1", 136 | "outputs": ["foo"], 137 | "inputs": None, 138 | }}, { 139 | "complete": False, 140 | "wrapper_arguments": { 141 | "job_id": "job 2", 142 | "outputs": None, 143 | "inputs": ["foo", "bar"], 144 | }}, { 145 | "complete": True, 146 | "wrapper_arguments": { 147 | "job_id": "job 3", 148 | "outputs": ["bar"], 149 | "inputs": None, 150 | }}) 151 | 152 | self.assertTrue( 153 | lib.run_printer.run_consistent_to_job(self.run, "job 2")) 154 | 155 | 156 | def test_one_of_two_incomplete(self): 157 | self.add_jobs({ 158 | "complete": True, 159 | "wrapper_arguments": { 160 | "job_id": "job 1", 161 | "outputs": ["foo"], 162 | "inputs": None, 163 | }}, { 164 | "complete": False, 165 | "wrapper_arguments": { 166 | "job_id": "job 2", 167 | "outputs": None, 168 | "inputs": ["foo", "bar"], 169 | }}, { 170 | "complete": False, 171 | "wrapper_arguments": { 172 | "job_id": "job 3", 173 | "outputs": ["bar"], 174 | "inputs": None, 175 | }}) 176 | 177 | with self.assertRaises(lib.run_printer.InconsistentRunError): 178 | lib.run_printer.run_consistent_to_job(self.run, "job 2") 179 | -------------------------------------------------------------------------------- /test/unit/status_parser.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | import pathlib 16 | import tempfile 17 | import unittest 18 | import unittest.mock 19 | 20 | import lib.ninja 21 | 22 | 23 | 24 | class TestNinjaStatusParser(unittest.TestCase): 25 | def setUp(self): 26 | self.sp = lib.ninja._StatusParser() 27 | 28 | 29 | def test_bad_status(self): 30 | self.assertEqual(None, self.sp.parse_status("foo bar")) 31 | 32 | 33 | def test_usual_ninja_status(self): 34 | self.assertEqual(None, self.sp.parse_status("[24/42] foo bar")) 35 | 36 | 37 | def test_expected_status(self): 38 | self.assertEqual({ 39 | "running": 34, 40 | "finished": 53, 41 | "total": 91, 42 | "message": "hello world" 43 | }, self.sp.parse_status(":34/53/91 hello world")) 44 | --------------------------------------------------------------------------------