├── .devcontainer.json ├── .dockerignore ├── .github └── workflows │ ├── codeql-analysis.yml │ ├── gitlab.yml │ ├── images.yml │ ├── release.yml │ ├── test-deploy.yml │ └── trigger-external.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── assets ├── demo.tf ├── logo.pdf ├── logo.png ├── test.md ├── test.svg ├── vega-lite.json └── watermark.svg ├── bin ├── cml.js ├── cml.test.js ├── cml │ ├── asset.js │ ├── asset │ │ ├── publish.e2e.test.js │ │ ├── publish.js │ │ └── publish.test.js │ ├── check.js │ ├── check │ │ ├── create.e2e.test.js │ │ ├── create.js │ │ └── create.test.js │ ├── comment.js │ ├── comment │ │ ├── create.e2e.test.js │ │ ├── create.js │ │ ├── create.test.js │ │ └── update.js │ ├── pr.js │ ├── pr │ │ ├── create.e2e.test.js │ │ └── create.js │ ├── repo.js │ ├── repo │ │ ├── prepare.js │ │ └── prepare.test.js │ ├── runner.js │ ├── runner │ │ ├── launch.e2e.test.js │ │ ├── launch.js │ │ └── launch.test.js │ ├── tensorboard.js │ ├── tensorboard │ │ ├── connect.e2e.test.js │ │ ├── connect.js │ │ └── connect.test.js │ ├── workflow.js │ └── workflow │ │ ├── rerun.js │ │ └── rerun.test.js └── legacy │ ├── commands │ ├── ci.js │ ├── publish.js │ ├── rerun-workflow.js │ ├── send-comment.js │ ├── send-github-check.js │ └── tensorboard-dev.js │ ├── deprecation.js │ ├── link.e2e.test.js │ └── link.js ├── package-lock.json ├── package.json ├── src ├── analytics.e2e.test.js ├── analytics.js ├── cml.e2e.test.js ├── cml.js ├── commenttarget.js ├── commenttarget.test.js ├── drivers │ ├── bitbucket_cloud.e2e.test.js │ ├── bitbucket_cloud.js │ ├── github.e2e.test.js │ ├── github.js │ ├── gitlab.e2e.test.js │ └── gitlab.js ├── logger.js ├── terraform.js ├── terraform.test.js ├── utils.js ├── utils.test.js ├── watermark.js └── watermark.test.js └── tests ├── proxy.js ├── setup.js └── teardown.js /.devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "image": "mcr.microsoft.com/vscode/devcontainers/javascript-node:16", 3 | "postCreateCommand": "npm ci && npm install --location=global", 4 | "hostRequirements": { 5 | "cpus": 4, 6 | "memory": "8gb", 7 | "storage": "32gb" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/node_modules -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: CodeQL 2 | 3 | on: 4 | pull_request: 5 | paths-ignore: 6 | - '**.md' 7 | - assets/** 8 | schedule: 9 | - cron: '0 0 * * *' # everyday @ 0000 UTC 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.ref_name }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | analyze: 17 | name: Analyze 18 | runs-on: ubuntu-latest 19 | permissions: 20 | actions: read 21 | contents: read 22 | security-events: write 23 | steps: 24 | - uses: actions/checkout@v3 25 | - uses: github/codeql-action/init@v2 26 | with: 27 | languages: javascript 28 | - uses: github/codeql-action/autobuild@v2 29 | - uses: github/codeql-action/analyze@v2 30 | -------------------------------------------------------------------------------- /.github/workflows/gitlab.yml: -------------------------------------------------------------------------------- 1 | name: GitLab 2 | on: pull_request 3 | jobs: 4 | gitlab: 5 | runs-on: ubuntu-latest 6 | services: 7 | gitlab: 8 | image: docker://gitlab/gitlab-ce 9 | ports: ['8000:8000'] 10 | env: 11 | GITLAB_OMNIBUS_CONFIG: | 12 | external_url 'http://localhost:8000/gitlab' 13 | nginx['custom_nginx_config'] = ' 14 | server { 15 | listen 8000; 16 | location /gitlab { 17 | proxy_set_header Host $http_host; 18 | proxy_pass http://gitlab-workhorse; 19 | } 20 | } 21 | ' 22 | steps: 23 | - uses: actions/checkout@v3 24 | - name: Configure credentials 25 | run: | 26 | docker exec ${{ job.services.gitlab.id }} bin/gitlab-rails runner " 27 | ; user = User.find_by_username('root') 28 | ; user.password = '${{ github.token }}' 29 | ; user.password_confirmation = '${{ github.token }}' 30 | ; user.save! 31 | ; token = user.personal_access_tokens.create(scopes: [:api], name: 'Token', expires_at: 1.days.from_now) 32 | ; token.set_token('${{ github.token }}') 33 | ; token.save! 34 | " 35 | - name: Create test project 36 | run: | 37 | curl "http://localhost:8000/gitlab/api/v4/projects" \ 38 | --header "PRIVATE-TOKEN: ${{ github.token }}" \ 39 | --request POST \ 40 | --get \ 41 | --data "name=test" 42 | - name: Create test commit 43 | run: | 44 | curl "http://localhost:8000/gitlab/api/v4/projects/root%2Ftest/repository/files/README.md" \ 45 | --header "PRIVATE-TOKEN: ${{ github.token }}" \ 46 | --request POST \ 47 | --get \ 48 | --data "author_email=test@test" \ 49 | --data "author_name=Test" \ 50 | --data "branch=main" \ 51 | --data "commit_message=Create%20README.md" \ 52 | --data "content=Test" 53 | - name: Get last commit 54 | id: commit 55 | run: | 56 | curl "http://localhost:8000/gitlab/api/v4/projects/root%2Ftest/repository/commits/main" \ 57 | --header "PRIVATE-TOKEN: ${{ github.token }}" \ 58 | --request GET \ 59 | | jq -r .id \ 60 | | xargs -0 printf "hash=%s" >> $GITHUB_OUTPUT 61 | - run: npm ci 62 | - name: Run cml-send-comment 63 | run: | 64 | node bin/cml.js send-comment \ 65 | --token=${{ github.token }} \ 66 | --repo=http://localhost:8000/gitlab/root/test \ 67 | --commit-sha=${{ steps.commit.outputs.hash }} \ 68 | --driver=gitlab \ 69 | <(echo message) 70 | -------------------------------------------------------------------------------- /.github/workflows/images.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: 3 | inputs: 4 | release: 5 | required: true 6 | type: boolean 7 | version: 8 | required: false 9 | type: string 10 | jobs: 11 | images: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | dvc: [1, 2, 3] 16 | base: [0, 1] 17 | gpu: [false, true] 18 | include: 19 | - base: 0 20 | ubuntu: 18.04 21 | python: 2.7 22 | cuda: 11.2.2 23 | cudnn: 8 24 | - base: 1 25 | ubuntu: 20.04 26 | python: 3.8 27 | cuda: 11.2.2 28 | cudnn: 8 29 | - latest: true # update the values below after introducing a new major version 30 | base: 1 31 | dvc: 3 32 | steps: 33 | - uses: actions/checkout@v3 34 | with: 35 | ref: ${{ github.event.pull_request.head.sha || github.ref }} 36 | fetch-depth: 0 37 | - name: Metadata 38 | id: metadata 39 | env: 40 | CML_VERSION: ${{ inputs.version }} 41 | run: | 42 | latest_tag=$(git describe --tags | cut -d- -f1) 43 | test -n "$CML_VERSION" && latest_tag="$CML_VERSION" 44 | cml_version=${latest_tag##v} 45 | dvc_version=$(python3 -c ' 46 | from distutils.version import StrictVersion as Ver 47 | from urllib.request import urlopen 48 | from json import load 49 | data = load(urlopen("https://pypi.org/pypi/dvc/json")) 50 | ver_pre = "${{ matrix.dvc }}".rstrip(".") + "." 51 | print( 52 | max( 53 | (i.strip() for i in data["releases"] if i.startswith(ver_pre)), 54 | default="${{ matrix.dvc }}", 55 | key=Ver 56 | ) 57 | )') 58 | echo cache_tag=${cml_version}-${dvc_version}-${{ matrix.base }}-${{ matrix.gpu }} >> $GITHUB_OUTPUT 59 | echo cml_version=$cml_version >> $GITHUB_OUTPUT 60 | tag=${cml_version//.*/}-dvc${{ matrix.dvc }}-base${{ matrix.base }} 61 | if [[ ${{ matrix.gpu }} == true ]]; then 62 | echo base=nvidia/cuda:${{ matrix.cuda }}-cudnn${{ matrix.cudnn }}-runtime-ubuntu${{ matrix.ubuntu }} >> $GITHUB_OUTPUT 63 | tag=${tag}-gpu 64 | else 65 | echo base=ubuntu:${{ matrix.ubuntu }} >> $GITHUB_OUTPUT 66 | fi 67 | 68 | TAGS="$( 69 | for registry in docker.io/{dvcorg,iterativeai} ghcr.io/iterative; do 70 | if [[ "${{ matrix.latest }}" == "true" ]]; then 71 | if [[ "${{ matrix.gpu }}" == "true" ]]; then 72 | echo "${registry}/cml:latest-gpu" 73 | else 74 | echo "${registry}/cml:latest" 75 | fi 76 | fi 77 | echo "${registry}/cml:${tag}" 78 | done | head -c-1 79 | )" 80 | 81 | # https://github.com/orgs/community/discussions/26288#discussioncomment-3876281 82 | # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings 83 | delim="$(openssl rand -hex 8)" 84 | echo "tags<<${delim}" >> $GITHUB_OUTPUT 85 | echo "${TAGS}" >> $GITHUB_OUTPUT 86 | echo "${delim}" >> $GITHUB_OUTPUT 87 | - uses: docker/setup-buildx-action@v2 88 | - uses: actions/cache@v3 89 | with: 90 | path: /tmp/.buildx-cache 91 | key: 92 | ${{ runner.os }}-buildx-${{ steps.metadata.outputs.cache_tag }}-${{ 93 | github.sha }} 94 | restore-keys: 95 | ${{ runner.os }}-buildx-${{ steps.metadata.outputs.cache_tag }}- 96 | - uses: docker/login-action@v2 97 | with: 98 | registry: docker.io 99 | username: ${{ vars.DOCKERHUB_USERNAME }} 100 | password: ${{ secrets.DOCKERHUB_PASSWORD }} 101 | - uses: docker/login-action@v2 102 | with: 103 | registry: ghcr.io 104 | username: ${{ github.repository_owner }} 105 | password: ${{ github.token }} 106 | - uses: docker/build-push-action@v3 107 | with: 108 | push: 109 | ${{ inputs.release || github.event_name == 'push' || 110 | github.event_name == 'schedule' || github.event_name == 111 | 'workflow_dispatch' }} 112 | context: ./ 113 | file: ./Dockerfile 114 | tags: | 115 | ${{ steps.metadata.outputs.tags }} 116 | build-args: | 117 | CML_VERSION=${{ steps.metadata.outputs.cml_version }} 118 | DVC_VERSION=${{ matrix.dvc }} 119 | PYTHON_VERSION=${{ matrix.python }} 120 | BASE_IMAGE=${{ steps.metadata.outputs.base }} 121 | pull: true 122 | cache-from: type=local,src=/tmp/.buildx-cache 123 | cache-to: type=local,dest=/tmp/.buildx-cache-new 124 | - name: Move cache 125 | # https://github.com/docker/build-push-action/issues/252 126 | # https://github.com/moby/buildkit/issues/1896 127 | run: | 128 | rm -rf /tmp/.buildx-cache 129 | mv /tmp/.buildx-cache-new /tmp/.buildx-cache 130 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | bump: 6 | type: choice 7 | required: true 8 | description: Bump version number 9 | options: [major, minor, patch] 10 | default: patch 11 | pull_request: 12 | types: [closed] 13 | jobs: 14 | bump: 15 | if: github.event_name == 'workflow_dispatch' 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v3 19 | - run: | 20 | git config --global user.name Olivaw[bot] 21 | git config --global user.email 64868532+iterative-olivaw@users.noreply.github.com 22 | git checkout -b bump/$(npm version ${{ github.event.inputs.bump }}) 23 | git push --set-upstream origin HEAD 24 | gh pr create --title "Bump version to $(git describe --tags)" --body "Approve me 🤖" 25 | gh pr merge --auto --squash 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.ADMIN_GITHUB_TOKEN }} 28 | release: 29 | if: github.event.pull_request.merged && startsWith(github.head_ref, 'bump/') 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v3 33 | - run: > 34 | gh release create --target ${{ 35 | github.event.pull_request.merge_commit_sha }} {--title=CML\ 36 | ,}"$(basename "$GITHUB_HEAD_REF")" --generate-notes --draft 37 | env: 38 | GITHUB_TOKEN: ${{ secrets.ADMIN_GITHUB_TOKEN }} 39 | package: 40 | needs: release 41 | secrets: inherit 42 | uses: ./.github/workflows/test-deploy.yml 43 | with: 44 | release: true 45 | -------------------------------------------------------------------------------- /.github/workflows/test-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Test & Deploy 2 | on: 3 | pull_request_target: 4 | workflow_dispatch: 5 | workflow_call: 6 | inputs: 7 | release: 8 | required: true 9 | type: boolean 10 | schedule: 11 | - cron: '0 8 * * 1' # M H d m w (Mondays at 8:00) 12 | jobs: 13 | authorize: 14 | environment: 15 | ${{ (github.event_name == 'pull_request_target' && 16 | github.event.pull_request.head.repo.full_name != github.repository) && 17 | 'external' || 'internal' }} 18 | runs-on: ubuntu-latest 19 | steps: 20 | - run: echo ✓ 21 | check-lock: 22 | needs: authorize 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@v3 26 | with: 27 | ref: ${{ github.event.pull_request.head.sha || github.ref }} 28 | - name: npm lock file is v2 29 | run: jq --exit-status .lockfileVersion==2 < package-lock.json 30 | lint: 31 | needs: authorize 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v3 35 | with: 36 | ref: ${{ github.event.pull_request.head.sha || github.ref }} 37 | - uses: actions/setup-node@v3 38 | - run: npm ci 39 | - run: npm run lint 40 | test: 41 | needs: authorize 42 | runs-on: ubuntu-latest 43 | steps: 44 | - uses: actions/checkout@v3 45 | with: 46 | ref: ${{ github.event.pull_request.head.sha || github.ref }} 47 | - uses: actions/setup-node@v3 48 | with: 49 | node-version: 16 50 | - uses: actions/setup-python@v2 51 | with: 52 | python-version: 3.9 53 | - run: | 54 | pip install tensorboard 55 | pip install -I protobuf==3.20.1 56 | - run: npm ci && npm install --global 57 | - run: npm run test 58 | env: 59 | GITHUB_TOKEN: ${{ github.token }} 60 | TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }} 61 | TEST_GITHUB_REPOSITORY: ${{ vars.TEST_GITHUB_REPOSITORY }} 62 | TEST_GITHUB_COMMIT: ${{ vars.TEST_GITHUB_COMMIT }} 63 | TEST_GITHUB_ISSUE: ${{ vars.TEST_GITHUB_ISSUE }} 64 | TEST_GITLAB_TOKEN: ${{ secrets.TEST_GITLAB_TOKEN }} 65 | TEST_GITLAB_REPOSITORY: ${{ vars.TEST_GITLAB_REPOSITORY }} 66 | TEST_GITLAB_COMMIT: ${{ vars.TEST_GITLAB_COMMIT }} 67 | TEST_GITLAB_ISSUE: ${{ vars.TEST_GITLAB_ISSUE }} 68 | TEST_BITBUCKET_TOKEN: ${{ secrets.TEST_BITBUCKET_TOKEN }} 69 | TEST_BITBUCKET_REPOSITORY: ${{ vars.TEST_BITBUCKET_REPOSITORY }} 70 | TEST_BITBUCKET_COMMIT: ${{ vars.TEST_BITBUCKET_COMMIT }} 71 | TEST_BITBUCKET_ISSUE: ${{ vars.TEST_BITBUCKET_ISSUE }} 72 | test-os: 73 | needs: authorize 74 | name: test-${{ matrix.system }} 75 | strategy: 76 | matrix: 77 | system: [ubuntu, macos, windows] 78 | runs-on: ${{ matrix.system }}-latest 79 | steps: 80 | - uses: actions/checkout@v3 81 | with: 82 | ref: ${{ github.event.pull_request.head.sha || github.ref }} 83 | - if: matrix.system == 'windows' 84 | uses: actions/setup-node@v3 85 | - name: install 86 | shell: bash 87 | run: | 88 | # https://github.com/npm/npm/issues/18503#issuecomment-347579469 89 | npm pack && npm install -g --no-save ./*cml*.tgz 90 | for cmd in '' runner publish pr; do 91 | cml $cmd --version 92 | done 93 | - if: matrix.system != 'windows' 94 | run: | 95 | for cmd in runner publish pr; do 96 | cml-$cmd --version 97 | done 98 | packages: 99 | needs: [lint, test, test-os] 100 | runs-on: ubuntu-latest 101 | outputs: 102 | version: ${{ steps.publish.outputs.version }} 103 | steps: 104 | - uses: actions/checkout@v3 105 | with: 106 | ref: ${{ github.event.pull_request.head.sha || github.ref }} 107 | - uses: actions/setup-node@v3 108 | with: 109 | registry-url: https://registry.npmjs.org 110 | - run: npm install 111 | - id: publish 112 | run: | 113 | VERSION=$(jq -r .version < package.json) 114 | echo "version=$VERSION" >> $GITHUB_OUTPUT 115 | npm install --dry-run @dvcorg/cml@$VERSION || 116 | npm ${{ inputs.release && 'publish' || 'publish --dry-run' }} 117 | env: 118 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 119 | # Step required "thanks" to https://github.com/actions/runner-images/issues/6283 120 | - uses: Homebrew/actions/setup-homebrew@3040c87fbdd10e6ec48d90bd23cb0abb6d66152d 121 | - run: brew install ldid 122 | - run: | 123 | cp node_modules/@npcz/magic/dist/magic.mgc assets/magic.mgc 124 | npx --yes pkg --no-bytecode --public-packages "*" --public package.json 125 | for cmd in '' runner publish pr; do build/cml-linux-x64 $cmd --version; done 126 | cp build/cml-linux{-x64,} 127 | cp build/cml-macos{-x64,} 128 | - if: inputs.release 129 | run: 130 | find build -type f | xargs gh release upload 131 | "${GITHUB_HEAD_REF#bump/}" 132 | env: 133 | GITHUB_TOKEN: ${{ github.token }} 134 | images: 135 | needs: packages 136 | secrets: inherit 137 | uses: ./.github/workflows/images.yml 138 | with: 139 | release: ${{ inputs.release || false }} 140 | version: ${{ needs.packages.outputs.version }} 141 | -------------------------------------------------------------------------------- /.github/workflows/trigger-external.yml: -------------------------------------------------------------------------------- 1 | name: Trigger external examples & test 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | tag: 6 | description: 'Tag to pass with repository_dispatch ex: "v0.15.1"' 7 | type: string 8 | required: true 9 | release: 10 | types: [published] 11 | pull_request_target: 12 | branches: [main] 13 | push: 14 | branches: [main] 15 | jobs: 16 | push: 17 | if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} 18 | runs-on: ubuntu-latest 19 | strategy: 20 | matrix: 21 | repos: [cml-playground] 22 | steps: 23 | - name: Trigger external actions 24 | run: | 25 | curl --silent --show-error --request POST \ 26 | --header "Authorization: token ${{ secrets.TEST_GITHUB_TOKEN }}" \ 27 | --header "Accept: application/vnd.github.v3+json" \ 28 | --url "https://api.github.com/repos/iterative/${{ matrix.repos }}/dispatches" \ 29 | --data '{"event_type":"push", "client_payload": {"branch":"main"}}' 30 | pr: 31 | if: ${{ github.event_name == 'pull_request_target' }} 32 | runs-on: ubuntu-latest 33 | strategy: 34 | matrix: 35 | repos: [cml-playground] 36 | steps: 37 | - name: Trigger external actions 38 | run: | 39 | curl --silent --show-error --request POST \ 40 | --header "Authorization: token ${{ secrets.TEST_GITHUB_TOKEN }}" \ 41 | --header "Accept: application/vnd.github.v3+json" \ 42 | --url "https://api.github.com/repos/iterative/${{ matrix.repos }}/dispatches" \ 43 | --data '{"event_type":"pr", "client_payload": {"branch":"${{ github.ref_name }}"}}' 44 | trigger: 45 | if: 46 | ${{ github.event_name == 'release' || github.event_name == 47 | 'workflow_dispatch' }} 48 | runs-on: ubuntu-latest 49 | strategy: 50 | matrix: 51 | repos: [cml-playground] 52 | steps: 53 | - name: Trigger external actions 54 | run: | 55 | curl --silent --show-error --request POST \ 56 | --header "Authorization: token ${{ secrets.TEST_GITHUB_TOKEN }}" \ 57 | --header "Accept: application/vnd.github.v3+json" \ 58 | --url "https://api.github.com/repos/iterative/${{ matrix.repos }}/dispatches" \ 59 | --data '{"event_type":"new-cml", "client_payload": {"tag":"${{ github.event.release.tag_name || github.event.inputs.tag }}"}}' 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Optional stylelint cache 58 | .stylelintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # dotenv environment variable files 76 | .env 77 | .env.development.local 78 | .env.test.local 79 | .env.production.local 80 | .env.local 81 | 82 | # parcel-bundler cache (https://parceljs.org/) 83 | .cache 84 | .parcel-cache 85 | 86 | # Next.js build output 87 | .next 88 | out 89 | 90 | # Nuxt.js build / generate output 91 | .nuxt 92 | dist 93 | 94 | # Gatsby files 95 | .cache/ 96 | # Comment in the public line in if your project uses Gatsby and not Next.js 97 | # https://nextjs.org/blog/next-9-1#public-directory-support 98 | # public 99 | 100 | # vuepress build output 101 | .vuepress/dist 102 | 103 | # vuepress v2.x temp and cache directory 104 | .temp 105 | .cache 106 | 107 | # Docusaurus cache and generated files 108 | .docusaurus 109 | 110 | # Serverless directories 111 | .serverless/ 112 | 113 | # FuseBox cache 114 | .fusebox/ 115 | 116 | # DynamoDB Local files 117 | .dynamodb/ 118 | 119 | # TernJS port file 120 | .tern-port 121 | 122 | # Stores VSCode versions used for testing VSCode extensions 123 | .vscode-test 124 | 125 | # yarn v2 126 | .yarn/cache 127 | .yarn/unplugged 128 | .yarn/build-state.yml 129 | .yarn/install-state.gz 130 | .pnp.* 131 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=ubuntu:20.04 2 | FROM ${BASE_IMAGE} 3 | ARG BASE_IMAGE 4 | 5 | LABEL maintainer="CML " 6 | 7 | # CONFIGURE NON-INTERACTIVE APT 8 | ENV DEBIAN_FRONTEND=noninteractive 9 | RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90assumeyes 10 | 11 | # CONFIGURE SHELL 12 | SHELL ["/bin/bash", "-c"] 13 | 14 | # FIX NVIDIA APT GPG KEYS (https://github.com/NVIDIA/cuda-repo-management/issues/1#issuecomment-1111490201) 🤬 15 | RUN grep nvidia <<< ${BASE_IMAGE} \ 16 | && for list in cuda nvidia-ml; do mv /etc/apt/sources.list.d/$list.list{,.backup}; done \ 17 | && apt-get update \ 18 | && apt-get install --yes gpg \ 19 | && apt-key del 7fa2af80 \ 20 | && apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub \ 21 | && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1404/x86_64/7fa2af80.pub \ 22 | && apt-get purge --yes gpg \ 23 | && apt-get clean \ 24 | && rm --recursive --force /var/lib/apt/lists/* \ 25 | && for list in cuda nvidia-ml; do mv /etc/apt/sources.list.d/$list.list{.backup,}; done \ 26 | || true 27 | 28 | # INSTALL CORE DEPENDENCIES 29 | RUN apt-get update \ 30 | && apt-get install --no-install-recommends \ 31 | build-essential \ 32 | apt-utils \ 33 | apt-transport-https \ 34 | ca-certificates \ 35 | iputils-ping \ 36 | software-properties-common \ 37 | pkg-config \ 38 | curl \ 39 | wget \ 40 | unzip \ 41 | gpg-agent \ 42 | sudo \ 43 | tzdata \ 44 | locales \ 45 | && locale-gen en_US.UTF-8 \ 46 | && apt-get clean \ 47 | && rm --recursive --force /var/lib/apt/lists/* 48 | 49 | # CONFIGURE LOCALE 50 | ENV LANG="en_US.UTF-8" 51 | ENV LANGUAGE="en_US:en" 52 | ENV LC_ALL="en_US.UTF-8" 53 | 54 | # INSTALL NODE, GIT & GO 55 | RUN add-apt-repository ppa:git-core/ppa --yes \ 56 | && add-apt-repository ppa:longsleep/golang-backports --yes \ 57 | && curl --location https://deb.nodesource.com/setup_16.x | bash \ 58 | && apt-get update \ 59 | && apt-get install --yes git golang-go nodejs \ 60 | && apt-get clean \ 61 | && rm --recursive --force /var/lib/apt/lists/* 62 | 63 | # INSTALL TERRAFORM 64 | RUN curl --remote-name --location https://releases.hashicorp.com/terraform/1.9.8/terraform_1.9.8_linux_amd64.zip \ 65 | && unzip terraform_1.9.8_linux_amd64.zip \ 66 | && mv terraform /usr/bin \ 67 | && rm LICENSE.txt terraform_1.9.8_linux_amd64.zip \ 68 | && terraform version # make sure it works 69 | 70 | # INSTALL LEO 71 | RUN curl --location https://github.com/iterative/terraform-provider-iterative/releases/latest/download/leo_linux_amd64 \ 72 | --output /usr/bin/leo \ 73 | && chmod +x /usr/bin/leo 74 | 75 | # INSTALL PYTHON 76 | ARG PYTHON_VERSION=3 77 | RUN add-apt-repository universe --yes \ 78 | && apt-get update \ 79 | && PYTHON_SUFFIX="$(sed --expression='s/3.*/3/g' --expression='s/2.*//g' <<< "${PYTHON_VERSION}")" \ 80 | && apt-get install --yes --no-install-recommends python${PYTHON_VERSION} python${PYTHON_SUFFIX}{-pip,-setuptools,-dev} \ 81 | && update-alternatives --install /usr/bin/python python${PYTHON_VERSION} $(which python${PYTHON_VERSION}) 10 \ 82 | && python -m pip install pip --upgrade \ 83 | && apt-get clean \ 84 | && rm --recursive --force /var/lib/apt/lists/* 85 | 86 | # INSTALL DVC 87 | ARG DVC_VERSION=3 88 | RUN cd /etc/apt/sources.list.d \ 89 | && wget https://dvc.org/deb/dvc.list \ 90 | && apt-get update \ 91 | && apt-get install --yes "dvc=${DVC_VERSION}.*" \ 92 | && apt-get clean \ 93 | && rm --recursive --force /var/lib/apt/lists/* 94 | 95 | # INSTALL CML 96 | ARG CML_VERSION=0 97 | RUN npm config set user 0 \ 98 | && npm install --global "@dvcorg/cml@${CML_VERSION}" 99 | 100 | # INSTALL VEGA 101 | RUN add-apt-repository universe --yes \ 102 | && apt-get update \ 103 | && apt-get install --yes \ 104 | libcairo2-dev \ 105 | libpango1.0-dev \ 106 | libjpeg-dev \ 107 | libgif-dev \ 108 | librsvg2-dev \ 109 | libfontconfig-dev \ 110 | && apt-get clean \ 111 | && rm --recursive --force /var/lib/apt/lists/* \ 112 | && npm config set user 0 \ 113 | && npm install --global canvas@2 vega@5 vega-cli@5 vega-lite@5.14.1 114 | 115 | # CONFIGURE RUNNER PATH 116 | ENV CML_RUNNER_PATH=/home/runner 117 | RUN mkdir ${CML_RUNNER_PATH} 118 | WORKDIR ${CML_RUNNER_PATH} 119 | 120 | # SET SPECIFIC ENVIRONMENT VARIABLES 121 | ENV IN_DOCKER=1 122 | ENV RUNNER_ALLOW_RUNASROOT=1 123 | # Environment variable used by cml to detect it's been installed using the docker image. 124 | ENV _CML_CONTAINER_IMAGE=true 125 | 126 | # DEFINE ENTRY POINT AND COMMAND 127 | # Smart entrypoint understands commands like `bash` or `/bin/sh` but defaults to `cml`; 128 | # also works for GitLab CI/CD 129 | # https://gitlab.com/gitlab-org/gitlab-runner/-/blob/4c42e96/shells/bash.go#L18-37 130 | # https://gitlab.com/gitlab-org/gitlab-runner/-/blob/4c42e96/shells/bash.go#L288 131 | ENTRYPOINT ["/bin/bash", "-c", "echo \"$0\" | grep -qE '^(pr|publish|runner|send-(comment|github-check)|tensorboard-dev|--?\\w.*)$' && exec cml \"$0\" \"$@\" || exec \"$0\" \"$@\""] 132 | CMD ["--help"] 133 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020-2021 Iterative, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /assets/demo.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | iterative = { 4 | versions = ["0.4.0"] 5 | source = "DavidGOrtega/iterative" 6 | } 7 | } 8 | } 9 | 10 | provider "iterative" {} 11 | 12 | resource "iterative_machine" "machine" { 13 | region = "us-west-1" 14 | } 15 | -------------------------------------------------------------------------------- /assets/logo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iterative/cml/a3a66c74b218b08710f42bf599b066e37cb4cba7/assets/logo.pdf -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iterative/cml/a3a66c74b218b08710f42bf599b066e37cb4cba7/assets/logo.png -------------------------------------------------------------------------------- /assets/test.md: -------------------------------------------------------------------------------- 1 | ### test 2 | 3 | ![embed](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADIAAAAcCAYAAAAjmez3AAABe2lDQ1BpY2MAACiRfZE9SMNAHMVfUyUqLQ5mEHHIUJ0siIo4ahWKUCHUCq06mI9+QZOGJMXFUXAtOPixWHVwcdbVwVUQBD9AnBydFF2kxP8lhRYxHhz34929x907gGtUVN3uGgd0w7HSyYSYza2K/Ct49EJAFLys2uacJKUQOL7uEWLrXZxlBZ/7c0S1vK0CIZF4VjUth3iDeHrTMRnvEwtqSdaIz4nHLLog8SPTFZ/fGBc95limYGXS88QCsVjsYKWD1ZKlE08RxzTdoHwu67PGeIuxXqmprXuyF0byxsoy02kOI4lFLEGCCAU1lFGBgzitBik20rSfCPAPeX6JXAq5ylDJsYAqdMieH+wPfndrFyYn/KRIAuh+cd2PEYDfBZp11/0+dt3mCRB+Bq6Mtr/aAGY+Sa+3tdgR0L8NXFy3NWUPuNwBBp9M2ZI9KUyTKxSA9zP6phwwcAv0rfm9tfZx+gBkqKvUDXBwCIwWKXs94N09nb39e6bV3w/2QHJ1PtCYugAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAACE3AAAhNwEzWJ96AAAAB3RJTUUH5AceDCAbVRwbogAAAcF6VFh0UmF3IHByb2ZpbGUgdHlwZSBpY2MAADiNpVNbjhwhDPznFDmC8bM5Tg80Uu5/gRgM89rZSJtYatGUsV02Rfpda/o1TA0TDMMTtCppMwKmCWnTy9hQkI0RQQ4pciKAXexu868A5PHfxj5pVjIy4CwgwBX+wbpXHYzyBhphuzP7oaUfnm/KKkYahTIumCF5Y2BoHA5dDlIznxBsvByBZ3a+h49j4ecaBdbk45xjDEd7BLzgV73j9oRvQo6PROw3E1Qx86oAHvAZ/+Z8GlQNtcWetoN9Fi6A1YKze8F547Zb4+RwNdH3W9o3WVX0EhHaAcvtCXyEwK4ldhIybm00qkNp4qs71A+NIWNb+yvODEkIrgT4TiD9nUE+HwwEVjGMYrMIRRFmT6R9PAj/8oMBPJ7Ekw0Fw5uAXVNKRPgsyAyVI1La1YcRytxbmQx7KzDx25nn2lYlanUmqs7iEwPt0RkdPRJS+cgUufRg1I95Gzet/DEhFpt+O+aaj+aAZpfNvc2RaE8e1W3mxdNiCFJmIIXM4JTbi+7CaCeCk9oMFA6Bca3zYCnlXBUicZf6ne72sN+uHfab+yK8/0j0KsT0B9wNICkQHDN8AAAKZElEQVRYw72YfbBdZXXGf2u979777HPOvTf3JpDQGBxkiC3FKo6QphqCIciX2AGmOLU4DvGzJJ0Siv6hQabFEj7GyihMa/0AzIBQKDCplpRqTDtTLWpJEUFjplVEEXEYIcm9556z97tW/9jnnOQGHOAP+s6s2XP2+VrP+6znWevdwmFrbv0KAglHUIxAmhTx1cCZwGrgGGDaNcZUdnvWnvxFKicfTu3JXVZ2v+aie+pFR5q1OoglrOiwbNNHeaWXHPri2fXHUhEpmSej6gh2ruIbBD9ZxKcO/axrJJUTWHuS1ERt5cQTqez+i7U62zwrHkydRameXILOz7J8w6b/HyBPnXY8AUNJ5FK9LpCuDKRzFcsVQ8QXfHEEJLUnsc5UA6acwMoJrOw+k1rdf7Ci/WkrOz/0rAWAFSWvPv+drxyQx087kUhNkET0+qwo9aci6bWBRCChGCr2fCBDNg4H0ly7WKuz14r2tZa3bvO8nMcMK7scc/Y5vzGhzbt/QEDwYXICqAiCoAI+THvr649bCGTPutW0pQc4ip8TqD8bqZdnUhNpQjEOZ2UMpDNFak8NS2wCKydJZRdrdbFWh1R2+l60v2R5+VeWFT+zVhdx49h1py5I5E93P0qGYEBAMJwC7QiUKuKCzAaReREIKI4TRfn4614DgFZk9MkZkL+xIt5QE5fXRGqPQz4CjuAHq9CAAwjPIdIHARFcBESH11EASOES3u8abifEN4kblhXs/Y8HxyDe+9AjDNyYxwDawOkZ+ingq8AuYKfAdodrcE5xvMg10I0ZWx/9ScPIY+veAjAVSLcF0jlBEpGajIpMKnIqIhWR9KRiO0R8F/ATD7Gf2otmUnfRG1J76m2pPbEqtSfbNiqvVgcbR8OO5a29nhWbXcNXPWthIXC1Niy3JCriayOyOaLrAnQCMi4tFUERgsi+ILo9iFwTRR9VAUWIwx2/wNAzDMV8uKsADiIkwf8xYNfgPDxk/6BPuO/wEG5Cw1o0bEL0NBeJiIIqaMBVcVHQcJxr+ILH/CMW4zYN0Qdpnkx0ekDaHF03ujDjY4U8X9AqMqnCRUHkjSpsFGRXwtCet5ZUZBdXZLFqeKDyjEFTbjbw7DM18X2JsBvBkisAYceTVIt/i8c/fBuI7nfVr3iIF3oIl6H6BKq4BFwDaKB5LSC6FA2fRsPG4xYfo+asNPebzX2L4TNGA2MUh4EYsqIEkeMV+dso8nuFROSbbz3nAsVvVayjYoycalhe92RSXdyivy+XAQV9Ol/76fN26tcf20DqTEGq8aJNak+ssqJzrbW6a1PZwYoOXpRYXuJZC8sLyIoD33G75b5qblVUPSmKEEXJUDIRMpSIoDBKnEyVXJSo2txDUJG7gsjFsSaeJXhHcNQNJRAwagm/qonXDTzfp2KIGzNf3/uCljn9118E4Om/2QJmeMgf9Ji9y0O8Gg0XoRpctDEFbUpuj9Xdf6/6mxzH3DHAcEx8aC7j8qoEJIjEbAgiDoGFhqGzROQMTYSTR0Zb0ZRUn5x5b+2a99ZDs95h3lss/foPXrQpHXnZJ0gTM6TuNB7zJz1mm1zjdYjON5ppSuyRuubu3iy/TDXuDZA0vI7CYZ/jNwMXIrwrim7PVC2XQK5KJg0gFekqcqEaurzp54FDAfUpvjlHu8qkYuXO/3xREKN11Acvx1WxVheP+QEP4UoP4QpU97so/1XV3Nfr8VydEDPcHDcbg0nuJPx7hr/bhQ8A90XRu8oQN5QaHyjGjCjK0M2Ek6OhkwtTcQRq4GcAc16+ZBCjteJP3gvAj7+ynRBjtaIbPrm3jrPfHqTr768GnVkVJOjw33xog46pIaL/2kv1xhiyvWn43hExYyJmzwSRe0HOPNzPBI6IPvbaEQw9xFvhGWZeNpDR+tjPM6qo7BGX3+72A5lKHRXJtFGEKIg1mhA4oZzh1OllurToaM+saarAT+cP8Fw1ILmLD9WzMGeIjuwDpgEMHc0yAVjhQN9bLxvAe7ZuxwrF+0YG+fG5brbKrjCkLRhIs2GC4S5oDJw8OcO66aNoxfy0nqW7EbkS+KfKvZqMOQ5L5lM6v3IbaugQQM7TaugTI43Y6ATS6OXNA4osSs2ta//8JYPYsOUecEfnE1LZpFR+lVT+l1J5R2pDah+GQW3EBGsmFnP69DJaIY428gSQW4AvCFwQRP+41HhrK4T1LQ1kI50gI518Ww19MI27RxiKPdL3fG3f8zfNeUnfcz5zypYXBfH+y+4izibigRqt/FVa+U1S2eVSWyG1o0MAI0C5CWunj+DUxcvIQ2zUOZ7RmADe7XCHwLZM9exCg+aq5BrIxv1E9qtwpybCjkSYrQnNVOUZA88ZkC/pU3yk5+VURU5NxtY117wggEs23M6HNt5J6CWoDUn+B1Lbl6X2i6R2lcrR2pDqICMk278ky29Yc8TSb0UNB0GM5Muou3sUIeiwYUZpGuaon0TRf1bkAbn31A2LzeVeQ9eMSmzMjEcbkN/U92JLj3LfHG0iNT1KBke28AAT/zPHvpUdPFNSGRZZLhdboX9huS63XElFwHLFC8UyxXLBc33WMv3o3Y889HePffCS19Si16F6nosKw8bZaBYqt6EtG7U7lRm1N5HcH0vu7wS+H8579aqeo54Ib6+JOhrhm7krlwH5SRX5ayuy/3X0l3Ox46kTQQUxGEznoDLpKmehcj1BPoDKlCt4M5o2IaOr/AKRzQ43r1y6zN969NG/dtUHXHQO1RMQ6YwYsaH9jueuBf4qjwCXCPLdVgzE1NjtPUY4PxHOrb2ZtKphY6w805p4QSK8pSY+AHwD+DHQB2YQTgROx/0kzEuSQ3LEBEk0pRQcT47UvkfUL/XkO1QgF8U0ADxrMfuEmv0byKXAeoeJw4fG4eD4rCL3BbjWXX6IOP2UkNvXXsLAcxx5QyLcWRNW1qORccxMM7pUZNRZdC9DzzNJCIWr5B4Fj4Jl0pzwF8T43i7P9DLLdbdniivc8b4zAXj4qWqYrYJICaw2ODO5n2j4kuTuhj9l7g+Z+/0G3xlYGvy8N8uyVpuzV3Sb88i8F4j6f7vrpTXh72viq2pvdDLSjKGY6OgA3V7QjhxwAQMxEPODkWRekt8itV+F+pMISD/x5T87eG5//bKMh59Oo9rpATsDsrPGS4dy+Otz3lQB7k6ugaPbXdYvbx+0h8+dcjkVGY5SEd9We7yhJv7OwVN7w0ytEcsVCsVjY5MjLTSs6CGsCJbpHs91q0W5wwrt42AtZduH3/EbLXz300bT66WZv/BxAzRvEn7zUfnzvjeWz42nbBk+DFIGnv9uIlxRE/+wJrZGpZZC40DksgAIKtiovKLimfzKcr3DMrnRivAjG302E275+Hkvubm+nLXABz655ipa9NjPBBVZOxHOrokbauLvJ4nTKTQ2SiZ4WAjEg9QW5XGPcr9F2YbId1MnWDURCfOJz1//R68IgBcEAnD1mmupiQhOIjAg7ybCqiThDIthtedyLJlMeZCI0HPlKUS+50G+4VF2kvxHdTe4ZwrueFQ+e+Mr81Du0PV/F8n/JQcwwagAABYCZVhJZklJKgAIAAAACQAAAQQAAQAAAPoAAAABAQQAAQAAAI8AAAACAQMAAwAAAHoAAAAaAQUAAQAAAIAAAAAbAQUAAQAAAIgAAAAoAQMAAQAAAAMAAAAxAQIADQAAAJAAAAAyAQIAFAAAAJ4AAABphwQAAQAAALIAAADEAAAACAAIAAgAVQAAAAEAAABVAAAAAQAAAEdJTVAgMi4xMC4yMAAAMjAyMDowNzowOSAxNjoxNDoxOQABAAGgAwABAAAAAQAAAAAAAAAIAAABBAABAAAAAAEAAAEBBAABAAAAkgAAAAIBAwADAAAAKgEAAAMBAwABAAAABgAAAAYBAwABAAAABgAAABUBAwABAAAAAwAAAAECBAABAAAAMAEAAAICBAABAAAA0RQAAAAAAAAIAAgACAD/2P/gABBKRklGAAEBAAABAAEAAP/bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAkLicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/AABEIAJIBAAMBIgACEQEDEQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0BAgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2gAMAwEAAhEDEQA/APn+iiigAoorf0Pwpe6uVlcGC1OD5jDlh/sionUjTXNJ2NqNCpXnyU1dmJBbzXUoit4ZJZDztjUsfyFdVpPgO8utsuoP9li67By5H8h+v0ruNN0ew0mPbaW6oSMM/Vm+p61oV49fMpvSmrfmfV4Ph6lC0sQ+Z9lt/mYVl4O0Sz2N9l8+RM/POxbOfVfu9/Sti3tbe0jMdtBFChOSsaBRn6CphRXmzqzn8TbPepYajRVqcUvRBT46ZT4+tQtzWWxY8qOaNopUV43BVlYZDA9QRWbceC/D18++TTIkbbtHkkxj8lIGffFasdWo67KU5R+F2PMxNKnU+OKfqecap8KZFV5NJvvMx92G4GCeOfnHGc9OB161wuo6NqWkyFL+ynt/m2hnT5WPs3Q/ga+i46lktoLuBoLiFJYnGGR1DAj3Br06WKmvi1Pn8TllJ609D5hor17xL8KIrpjc6C6W7Y+a2kJ2sfUHnH06fSvKb2wu9NuWtr23kgmXqki4Ppkeo4613QmprQ8SrQnSdpFeiiirMQooooAKKKKACiiigAooooAKKKKACiiigAooooAKKK3/AArof9rX/mSgfZoCC4I+8ew/xqZyUIuTNaNGVaoqcN2XfC3hY37Le3yEWoOUQ/8ALT/63869GQBVCqAAOAB2pigIoVQAo4AFPFeFiKkqsrs+7wOEp4Snyw36vuPpaYDTq4pI9KLFpaSlrMsKkjqOpI6I7ilsWo6tx1Vjq3HXXTPOqlmOrUYqtHVpK64Hm1SwlZ2v+F9N8S2D299Apk24jmA+eM+oNaKVZSuqGh51VJqzPnLxX4D1bwq7Syr9osMqBdIMDJ7EZyOfw6euK5avrie0gvbWS2uYklhlUq6OMhga8S+IPwyfQ1/tPRI5JdPwBLDks0J9fUqf0+nTtjK+549ajy6x2PNKKKKs5wooooAKKK6nw94B1vxAyukBtrUnmeYYGOOg6nrmmk3oiJzjBXk7HLUV7ZpHwe0m2UNqdzNeSFSGVD5aZzwRjnp7101r4C8MWlusK6PbSBc/NMu9jznknk1qqEnuccswpLa7Pm2ivpWbwN4YnheI6LaIGGNyRhWH0PasO9+EPhy4iVbY3NowOSySFsj0+bNDoSCOYU3umjwaiu78R/CzWdGV7iy/0+2GSfLGHUcnkd+B2/KuGdGjdkdSrqSGVhgg+hrJxcdzshUjUV4u42iiikWPiiknmSKJC8jnCqByTXr+l6fFpenxWkOSqDlj1JPU1w/gfTWn1F79gRHbjap9XIx+g/mK9CzXFiXzPlPpcloKEHWe729B4NKDTKUGuGUD34yJAacDUYNc54l8VJo4+zWuyS9OCQeVjHv7n0/H0ziqEpy5Yjq4qnQg6lR2Rsanren6Qmbu4VXxlYxyx/D8Otcdf/EK5clbC1SJcnDyncSPoOh/E1yFzdT3ly9xcSNJK5yzN3qKvRo5dShrPVny+Lz7EVXal7sfx/r0NO58Ravdspl1CfKjA2Ns/wDQcUyHXdVglWWPUbncpyN0hYfkeDWfRXYqUErKKPIeJrN8zm7+rOw034i6rZ4W7jivEGck/Ix/EcfpXfaB410jWSkQm+z3LYHkzcEnjoeh57dfavEaKxnhKUtlY7KOa4inpJ8y8/8AM+nI+QKtJXgvhvx9qnh9fIcfbbTjEUrncgAwArc4HTjBHHGM17RoOv6d4gslubC4V/lBeIkb4yezDt0P1xxmuaVCVPfY9OnjKddaaPsbSCrKCoEFWUFaQRjUZMgqbYrqVZQykYIIyCKjQVMorpijhqM8h+JfwxEnna7oMOH5e5tUHX1dR/MfjXixBBIIwR1Br7NWvJviJ8J31Gd9W8OxIJ2yZ7UYUMf7y+/tWyOKcdbo8Lq7pekX2s3YtrC3eaXqQo6D1NaHh3wrqHiDWDYxxtEsR/0iR1wIhnBz79eK938PeG9O8N2ht7CMgucvI5yzH3Na06Tnr0PNxeNjQ91ayOe8J/DSx0Vo7vUCt1eqcr/cTj0713ygKoVQABwAO1MBp4rsUVFWR4U606suabuPFLTRTqY0KKdSCnCkUhQK5Xxb8P8AS/FEDy7BbagqERzoMDOc/MO46/ma6wCpAKTSaszanKUHeLPkzVtJvdE1KbT7+ExXERwQehHYg9wfWqVfS3jzwdD4r0VxHEn9pwrm2kJ2/VSfQ/z/ABr5vurWeyupbW5iaKeJiro4wVI7GuOpDlZ7eHrqrHzPSfCdp9k8PW+U2PNmVuc5z0P/AHztrbzUNvCltbRQR52RIEXPoBgVLXnyV3c+7ox9nTjBdEOzS5pmaXNZuBuplXVdTi0nT5LuUZ28KoOCxPQV5Jc3M15cyXFxIZJZDlmPeuq8d3xkureyVjtjXzGGeCTwPxGD+dchXRQpqKv1Z81m2KdWr7NbR/MKKKK6DyQooooAKKKKACrulavfaJfLe6fcNDOoK5HIIPYg8EfX0qlRQ1cabTuj6D8GePLDxLHHbSkQalt+eI9HI6lT/Su4QV8kQTy2txHPBI0csbBkdTgg17t4F+JVprUcGn6o4h1I/KGPCy+/sawdO2qO+niedWnuekJU61CoqdRVxFNkqiia4jtojJIcDsPWori5jtIi8h+g7mudubuS6lLueOw7CuqlTcteh42Px0aC5VrIWV1knkkVAm9ixA9TQDUQNPBrstY+WlJybk+pKDUgqEGpFNBUWSinVE0qRIXkdUQdWY4ArhPEPxW0nSw0Omj7fcDuDiMH3PfvUyko7nVSpTqO0Vc9CFSKK8p0b4y208yRatYG3DPjzYW3Kox1IPPWvUrC7ttRs4ru0mSaCQbldTkEUozUtjadCdN++iwBUgFKFpwFMEhAK8O+Mvhf7HqUWvWsWIbn5J9q8LIOhOB39Sck17piub8d6K+v+D76yhtzPclQ8CBtuXB45yB69aipHmidFCfJNM4OikorybH6cLRSUUWC55Z4jmS48QXkkZJXft5HcAA/qKy60Ndtzba5eRlgx8wtkf7Xzf1rProWx8fXu6sr73YUUUUzIKKKKACiiigAooooAKVWZGDKxVgcgg4INJRQB7Z8O/if9se00PWA7XLt5cVzxhhjgN7+9esXN3HZw73PPZe5r5u8FeD5NWnTULwNHZxsGQDgyEentXshldwodi20YGTnArelh3LV7HLicx9mnCOrLc91JdSmSQ/QelMBqEGng12JW0R83Ubk3KW5MDTwahBp4NBi0TKao6zr2n+H7H7XqM4jjLBVAGWY+gHet7S9Le8YSSArCP8Ax6qPj74bWXjSxgEUv2O+tuIpsbl2k8qy9/8AGs5ystD0MLg3UtKeiPn7xZ451HxNcyxiR4NOJGy2B6gZwW9Tz9Pyrlq1Ne8O6p4b1GWx1S0eGRHKh8HY+MHKt0IwQfxrLrhk23qfQ04RhHlgtAr1v4Ja7Kt/daE+5onQ3EZ/uEYDfnkfrXklem/BPSprnxVPqHl5t7aAqWOR87EYx69P1FVTvzKxGISdN3Pe8UuKfijFdh5FhmKXFLijFAWPGUkSSNZI2DIwBVgcgg96dWX4duPtPh+yfbtxH5eM5+78uf0zWnXl8p+mwqc8FLuFFJmkzVKIOR5/40s/I1dbhR8twmSc9WHB/TFc3XpviLSzqumMkYHnxnfGTx9RXmVXax85j6XJWb6MKKKKDiCiiigAooooAKKKACTgDJoAK7fwr4Ha/WK+1LKW5O5YccuPf2q14W8FIEivtUU78hkgPb03f4V6EhAAA4Arso4f7Uzkr1mlyxJ4USGNY41CIowqjoBU6mqymplNdVjyJxLCmng1ApqQGkc8okwNbekaQ10RNMCIh0H96o9F0c3e24n4hB4X+9/9autRQqhVAAHAArKc+iOzC4Pm9+ewqKqKFUAKOABThQKUVmj0mZPiHw1pfinTDYatbCaHcGUg4ZD6g9q8Q8S/AfVLSVpdBuUvIME+XKdrrgfr7V9DUhqZQUtxqbjsfMehfBbxNqUkEmoRJYWzN+88xh5igH+779q9y8L+E7Dwjo66fYBmBO6SVvvSN6mulNRsKcYKOxnVnKejKpWmEVYK1GVrQ5nEixWV4k1f+wPD17qohExto94jLbd3tnBxWwRXj3xv8RxR2Vt4ehKNLIwnn6EoB90deCevI6VM5cquOnDmmkcD4H1BV8/T3wCx81D68AEfoP1rs815HY3kmn3sV1FjfGcgHv2I/KvVra4juraOeI5SRQwNcaVz7PL6/NT5HuvyJaTNGaaTWkYna5Ck1xnibw7K1w9/ZIZA5zJEo5B9QB1z375/TsSaaTW6pKSscmIjGrHlkeRUV3ur+Gba/LTQYguGJZiB8rfUdvrXKXmhajZE77dnX+/H8w/xrGdCcemh41SjKDM2iiisTIKKu2mkahesogtJWDDIYrhSPqeK6jTPA4DCTUptwB/1UR4P1P8Ah+dawoznsh2OUsdNvNSl8qzt3lYdcdB9SeB0PWvSfDnhS20fZcynzr3bgsfuoe+0fpn+WcVqWdrBZW6QW0SxxqMBV/z1q4prupYaMNXqzKo3sTqamU1XU1KprdnFNFhTUqmq6mpVNSzlnEsKawPE3jG08ORrHs8+7cZWINjaPU/4VU8VeLoNAtjDAVkv3HyJ1Ce5/wAK8gurqe9uXuLiRpJXOWZjya5a1bl0juaYfCe0fNPY9N8A/Fu+0rWpIdenefTbqTJJ5+zN0yo/u+o/HrnP0Jpuo2erWMV7YXCXFtKMpIh4NfFFdh4I+Ieq+C5ykB8+xkYGW3c8e5X0Ncsaltz05U1b3T6zpwrnfC/jPRvFtn5+m3ILg7Whf5XU49O9dFXQtTnYU006mmmSxhphFPNNNBDIiKYRUpFZGv8AiLSvDdibvVbtIIznYD95yBnCjueKdyLX2IfEmvWvhnQrnVbsbkhX5YwwBkY9FGf84zXylrutXXiDWrnU7xyZZ3zgkkKOyjPYVseN/G9/4z1UzSlobGIkW9sDwg9T6se5rlq5ak+Z6HbRpciu9wrqvCeuLbN/Z9y+I2P7knopPUfjXK0AkEEHBHeoTszrpVZUp80T2DNITXNeHfEYvFW0u2AuAMKx6OP8a6QmuumlJXR7ca0akeaIhNNNKaSuqESJMQ0nWg0orpijmmxslla3ShZ7eKQDkB1BxRBpljbSeZDZwRvjG5UANWEp9JxV72OWW4qgDgDAqQVGKeKCJEqmpVNQLUqmkzmmidTUimoVNLLPFbQvNM6pGgyzMcACpZzTRaU1yXi3xkumxNY6dIrXjD5pByIh/jWH4i8cvdLJaaWSkLDDT8hj649PrXE1w1sR9mA6eHu7yJJ55bq4knnkaSWRizM3Umo6KK4jrCiiigCzp+o3mlXkd5Y3ElvcR/dkQ4Ir2Lwj8dpYFjtPEtuZUzj7XCPmHI5Ze+BnpzXilFUpNbEyinufX+j/ABA8La75C2WsWxmm3bIJG2ScZzlTyOAT9K3o7q3mbbFPFI2M4RwTXxFVi0v7zT5Gksrue2dhgtDIUJHpkGtFW7oydHsz7Xd1jUu7BVHUscAVlal4l0TSIBNf6pawRscKWlHJxnAr5Gm1/WbiF4Z9Wv5YnGGR7l2Vh6EE81nU3W7ISod2e9+J/jrYQwmHw7bPczMvE86lEQnP8J5JBx7e9eM694j1XxNffa9Vu3nkGQinhYwTnCjsP/rVlUVlKbluaxpxjsFFFFSWFFFFACqxVgykgg5BHaus0jxaflg1H2CyqP8A0KuSoq4TcHdGlOpKm7xPWlYOoZSCD0IorzTT9YvdNb9xLlP+eb8r+VdRY+LrOZMXatbuB1ALKfy5rvpYiEt9DsjiYy30Oipy1DFNFcIJIZEkQ9GRgRU613LYJMkWn01adSMGLThTaGkSNGeRlVFGWZjgAetIhkwqRa52+8W6VZBgk32iUdFi5B4/vdMVymoeMtTvVaOIraxk/wDLP72PTd/hiuepiKcOtzGSudvqninTdJ3I8vmzj/llHyR9fT8a891jxDf61J+/k2QjgQoSF69/U1lMxZizElicknvSVwVa8qmnQlRSCiiisCgooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAfFLJDIJIpGjcdGQ4I/Gu68I3E1zpszzzSSsJiAXYsQNo9aKK78EbUjpBTqKK9AtmT4nmlt9AuJIZHjkBXDIxBHzDuK83nuZ7pw9xNJK4GA0jFjj05oorgxhlMiooorzzMKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//ZAHWWGjwAAAAldEVYdGRhdGU6Y3JlYXRlADIwMjAtMDctMzBUMTI6MzI6MDIrMDA6MDDi40UVAAAAJXRFWHRkYXRlOm1vZGlmeQAyMDIwLTA3LTMwVDEyOjMyOjAyKzAwOjAwk779qQAAABp0RVh0ZXhpZjpCaXRzUGVyU2FtcGxlADgsIDgsIDgS7T4nAAAAEXRFWHRleGlmOkNvbG9yU3BhY2UAMQ+bAkkAAAAhdEVYdGV4aWY6RGF0ZVRpbWUAMjAyMDowNzowOSAxNjoxNDoxOWHSjSwAAAATdEVYdGV4aWY6RXhpZk9mZnNldAAxNzjc1lZ+AAAAFHRFWHRleGlmOkltYWdlTGVuZ3RoADE0M4D8vbsAAAATdEVYdGV4aWY6SW1hZ2VXaWR0aAAyNTDR1HOUAAAAGnRFWHRleGlmOlNvZnR3YXJlAEdJTVAgMi4xMC4yMAYQhsIAAAAkdEVYdGV4aWY6dGh1bWJuYWlsOkJpdHNQZXJTYW1wbGUAOCwgOCwgOCAb9FMAAAAcdEVYdGV4aWY6dGh1bWJuYWlsOkNvbXByZXNzaW9uADb5ZXBXAAAAHnRFWHRleGlmOnRodW1ibmFpbDpJbWFnZUxlbmd0aAAxNDZLLb8bAAAAHXRFWHRleGlmOnRodW1ibmFpbDpJbWFnZVdpZHRoADI1NogG+hQAAAAodEVYdGV4aWY6dGh1bWJuYWlsOkpQRUdJbnRlcmNoYW5nZUZvcm1hdAAzMDSsR89oAAAAL3RFWHRleGlmOnRodW1ibmFpbDpKUEVHSW50ZXJjaGFuZ2VGb3JtYXRMZW5ndGgANTMyOY6ptrMAAAAqdEVYdGV4aWY6dGh1bWJuYWlsOlBob3RvbWV0cmljSW50ZXJwcmV0YXRpb24ANhIVihoAAAAgdEVYdGV4aWY6dGh1bWJuYWlsOlNhbXBsZXNQZXJQaXhlbAAz4dfNWgAAABt0RVh0aWNjOmNvcHlyaWdodABQdWJsaWMgRG9tYWlutpExWwAAACJ0RVh0aWNjOmRlc2NyaXB0aW9uAEdJTVAgYnVpbHQtaW4gc1JHQkxnQRMAAAAVdEVYdGljYzptYW51ZmFjdHVyZXIAR0lNUEyekMoAAAAOdEVYdGljYzptb2RlbABzUkdCW2BJQwAAAABJRU5ErkJggg==) 4 | -------------------------------------------------------------------------------- /assets/test.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /assets/vega-lite.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://vega.github.io/schema/vega-lite/v4.json", 3 | "description": "A simple bar chart with embedded data.", 4 | "data": { 5 | "values": [ 6 | { "a": "A", "b": 28 }, 7 | { "a": "B", "b": 55 }, 8 | { "a": "C", "b": 43 }, 9 | { "a": "D", "b": 91 }, 10 | { "a": "E", "b": 81 }, 11 | { "a": "F", "b": 53 }, 12 | { "a": "G", "b": 19 }, 13 | { "a": "H", "b": 87 }, 14 | { "a": "I", "b": 52 } 15 | ] 16 | }, 17 | "mark": "bar", 18 | "encoding": { 19 | "x": { "field": "a", "type": "nominal", "axis": { "labelAngle": 0 } }, 20 | "y": { "field": "b", "type": "quantitative" } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /assets/watermark.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /bin/cml.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { basename } = require('path'); 4 | const { pseudoexec } = require('pseudoexec'); 5 | 6 | const kebabcaseKeys = require('kebabcase-keys'); 7 | const which = require('which'); 8 | const { logger, setupLogger } = require('../src/logger'); 9 | const yargs = require('yargs'); 10 | 11 | const CML = require('../src/cml').default; 12 | const { jitsuEventPayload, send } = require('../src/analytics'); 13 | 14 | const aliasLegacyEnvironmentVariables = () => { 15 | const legacyEnvironmentPrefixes = { 16 | CML_CI: 'CML_REPO', 17 | CML_PUBLISH: 'CML_ASSET', 18 | CML_RERUN_WORKFLOW: 'CML_WORKFLOW', 19 | CML_SEND_COMMENT: 'CML_COMMENT', 20 | CML_SEND_GITHUB_CHECK: 'CML_CHECK', 21 | CML_TENSORBOARD_DEV: 'CML_TENSORBOARD' 22 | }; 23 | 24 | for (const [oldPrefix, newPrefix] of Object.entries( 25 | legacyEnvironmentPrefixes 26 | )) { 27 | for (const key in process.env) { 28 | if (key.startsWith(`${oldPrefix}_`)) 29 | process.env[key.replace(oldPrefix, newPrefix)] = process.env[key]; 30 | } 31 | } 32 | 33 | // Remap environment variable prefixes so e.g. CML_OPTION global options become 34 | // an alias for CML_COMMAND_OPTION, to be interpreted by the appropriate subcommands. 35 | // See also https://github.com/yargs/yargs/issues/873#issuecomment-917441475 36 | for (const globalOption of ['DRIVER', 'DRIVER_TOKEN', 'LOG', 'REPO', 'TOKEN']) 37 | for (const subcommand of [ 38 | 'ASSET', 39 | 'CHECK', 40 | 'COMMENT', 41 | 'PR', 42 | 'REPO', 43 | 'RUNNER', 44 | 'TENSORBOARD', 45 | 'WORKFLOW' 46 | ]) 47 | if (process.env[`CML_${globalOption}`] !== undefined) 48 | process.env[`CML_${subcommand}_${globalOption}`] = 49 | process.env[`CML_${globalOption}`]; 50 | 51 | const legacyEnvironmentVariables = { 52 | TB_CREDENTIALS: 'CML_TENSORBOARD_CREDENTIALS', 53 | DOCKER_MACHINE: 'CML_RUNNER_DOCKER_MACHINE', 54 | RUNNER_IDLE_TIMEOUT: 'CML_RUNNER_IDLE_TIMEOUT', 55 | RUNNER_LABELS: 'CML_RUNNER_LABELS', 56 | RUNNER_SINGLE: 'CML_RUNNER_SINGLE', 57 | RUNNER_REUSE: 'CML_RUNNER_REUSE', 58 | RUNNER_NO_RETRY: 'CML_RUNNER_NO_RETRY', 59 | RUNNER_DRIVER: 'CML_RUNNER_DRIVER', 60 | RUNNER_REPO: 'CML_RUNNER_REPO', 61 | RUNNER_PATH: 'CML_RUNNER_PATH' 62 | }; 63 | 64 | for (const [oldName, newName] of Object.entries(legacyEnvironmentVariables)) { 65 | if (process.env[oldName]) process.env[newName] = process.env[oldName]; 66 | } 67 | }; 68 | 69 | const setupOpts = (opts) => { 70 | const { markdownfile } = opts; 71 | opts.markdownFile = markdownfile; 72 | opts.cml = new CML(opts); 73 | }; 74 | 75 | const setupTelemetry = async (opts, yargs) => { 76 | const { cml, _: command } = opts; 77 | 78 | const options = {}; 79 | for (const [name, option] of Object.entries(opts.options)) { 80 | // Skip options with default values (i.e. not explicitly set by users) 81 | if (opts[name] && !yargs.parsed.defaulted[name]) { 82 | switch (option.telemetryData) { 83 | case 'name': 84 | options[name] = null; 85 | break; 86 | case 'full': 87 | options[name] = opts[name]; 88 | break; 89 | } 90 | } 91 | } 92 | 93 | opts.telemetryEvent = await jitsuEventPayload({ 94 | action: command.join(':'), 95 | extra: { options }, 96 | cml 97 | }); 98 | }; 99 | 100 | const runPlugin = async ({ $0: executable, command }) => { 101 | if (command === undefined) throw new Error('no command'); 102 | const { argv } = process.argv; 103 | const path = which.sync(`${basename(executable)}-${command}`); 104 | const parameters = argv.slice(argv.indexOf(command) + 1); // HACK 105 | await pseudoexec(path, parameters); 106 | }; 107 | 108 | const handleError = (message, error) => { 109 | if (!error) { 110 | yargs.showHelp(); 111 | console.error('\n' + message); 112 | process.exit(1); 113 | } 114 | }; 115 | 116 | (async () => { 117 | aliasLegacyEnvironmentVariables(); 118 | setupLogger({ log: 'debug' }); 119 | 120 | try { 121 | await yargs 122 | .options( 123 | kebabcaseKeys({ 124 | log: { 125 | type: 'string', 126 | description: 'Logging verbosity', 127 | choices: ['error', 'warn', 'info', 'debug'], 128 | default: 'info', 129 | group: 'Global Options:' 130 | }, 131 | driver: { 132 | type: 'string', 133 | choices: ['github', 'gitlab', 'bitbucket'], 134 | defaultDescription: 'infer from the environment', 135 | description: 'Git provider where the repository is hosted', 136 | group: 'Global Options:' 137 | }, 138 | repo: { 139 | type: 'string', 140 | defaultDescription: 'infer from the environment', 141 | description: 'Repository URL or slug', 142 | group: 'Global Options:' 143 | }, 144 | driverToken: { 145 | type: 'string', 146 | alias: 'token', 147 | defaultDescription: 'infer from the environment', 148 | description: 'CI driver personal/project access token (PAT)', 149 | group: 'Global Options:' 150 | } 151 | }) 152 | ) 153 | .global('version', false) 154 | .group('help', 'Global Options:') 155 | .fail(handleError) 156 | .middleware(setupOpts) 157 | .middleware(setupLogger) 158 | .middleware(setupTelemetry) 159 | .commandDir('./cml') 160 | .commandDir('./legacy/commands') 161 | .command( 162 | '$0 ', 163 | false, 164 | (builder) => builder.strict(false), 165 | runPlugin 166 | ) 167 | .recommendCommands() 168 | .demandCommand() 169 | .strict() 170 | .parse(); 171 | 172 | const { telemetryEvent } = yargs.parsed.argv; 173 | await send({ event: telemetryEvent }); 174 | } catch (err) { 175 | if (yargs.parsed.argv) { 176 | const { telemetryEvent } = yargs.parsed.argv; 177 | const event = { ...telemetryEvent, error: err.message }; 178 | await send({ event }); 179 | } 180 | logger.error(err); 181 | process.exit(1); 182 | } 183 | })(); 184 | -------------------------------------------------------------------------------- /bin/cml.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../src/utils'); 2 | const fetch = require('node-fetch'); 3 | 4 | describe('command-line interface tests', () => { 5 | test('cml --help', async () => { 6 | const output = await exec('node', './bin/cml.js', '--help'); 7 | 8 | expect(output).toMatchInlineSnapshot(` 9 | "cml.js 10 | 11 | Commands: 12 | cml.js check Manage CI checks 13 | cml.js comment Manage comments 14 | cml.js pr Manage pull requests 15 | cml.js runner Manage self-hosted (cloud & on-premise) CI runners 16 | cml.js workflow Manage CI workflows 17 | cml.js ci Prepare Git repository for CML operations 18 | 19 | Global Options: 20 | --log Logging verbosity 21 | [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] 22 | --driver Git provider where the repository is hosted 23 | [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the 24 | environment] 25 | --repo Repository URL or slug 26 | [string] [default: infer from the environment] 27 | --driver-token, --token CI driver personal/project access token (PAT) 28 | [string] [default: infer from the environment] 29 | --help Show help [boolean] 30 | 31 | Options: 32 | --version Show version number [boolean]" 33 | `); 34 | }); 35 | }); 36 | 37 | describe('Valid Docs URLs', () => { 38 | test.each([ 39 | 'workflow/rerun', 40 | 'tensorboard/connect', 41 | 'runner/launch', 42 | 'repo/prepare', 43 | 'pr/create', 44 | 'comment/create', 45 | 'comment/update', 46 | 'check/create', 47 | 'asset/publish' 48 | ])('Check Docs Link', async (cmd) => { 49 | const { DOCSURL } = require(`./cml/${cmd}`); 50 | const { status } = await fetch(DOCSURL); 51 | expect(status).toBe(200); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /bin/cml/asset.js: -------------------------------------------------------------------------------- 1 | exports.command = 'asset'; 2 | exports.description = false; 3 | exports.builder = (yargs) => 4 | yargs 5 | .commandDir('./asset', { exclude: /\.test\.js$/ }) 6 | .recommendCommands() 7 | .demandCommand() 8 | .strict(); 9 | -------------------------------------------------------------------------------- /bin/cml/asset/publish.e2e.test.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const { exec } = require('../../../src/utils'); 3 | 4 | describe('CML e2e', () => { 5 | test('cml publish assets/logo.png --md', async () => { 6 | const output = await exec( 7 | 'node', 8 | './bin/cml.js', 9 | 'publish', 10 | 'assets/logo.png', 11 | '--md' 12 | ); 13 | 14 | expect(output.startsWith('![](')).toBe(true); 15 | }); 16 | 17 | test('cml publish assets/logo.png', async () => { 18 | const output = await exec( 19 | 'node', 20 | './bin/cml.js', 21 | 'publish', 22 | 'assets/logo.png' 23 | ); 24 | 25 | expect(output.startsWith('https://')).toBe(true); 26 | }); 27 | 28 | test('cml publish assets/logo.pdf --md', async () => { 29 | const title = 'this is awesome'; 30 | const output = await exec( 31 | 'node', 32 | './bin/cml.js', 33 | 'publish', 34 | 'assets/logo.pdf', 35 | '--md', 36 | '--title', 37 | title 38 | ); 39 | 40 | expect(output.startsWith(`[${title}](`)).toBe(true); 41 | }); 42 | 43 | test('cml publish assets/logo.pdf', async () => { 44 | const output = await exec( 45 | 'node', 46 | './bin/cml.js', 47 | 'publish', 48 | 'assets/logo.pdf' 49 | ); 50 | 51 | expect(output.startsWith('https://')).toBe(true); 52 | }); 53 | 54 | test('cml publish assets/test.svg --md', async () => { 55 | const title = 'this is awesome'; 56 | const output = await exec( 57 | 'node', 58 | './bin/cml.js', 59 | 'publish', 60 | 'assets/test.svg', 61 | '--md', 62 | '--title', 63 | title 64 | ); 65 | 66 | expect(output.startsWith('![](') && output.endsWith(`${title}")`)).toBe( 67 | true 68 | ); 69 | }); 70 | 71 | test('cml publish assets/test.svg', async () => { 72 | const output = await exec( 73 | 'node', 74 | './bin/cml.js', 75 | 'publish', 76 | 'assets/test.svg' 77 | ); 78 | 79 | expect(output.startsWith('https://')).toBe(true); 80 | }); 81 | 82 | test('cml publish assets/logo.pdf to file', async () => { 83 | const file = `cml-publish-test.md`; 84 | 85 | await exec( 86 | 'node', 87 | './bin/cml.js', 88 | 'publish', 89 | 'assets/logo.pdf', 90 | '--file', 91 | file 92 | ); 93 | 94 | expect(fs.existsSync(file)).toBe(true); 95 | await fs.promises.unlink(file); 96 | }); 97 | 98 | test('cml publish assets/vega-lite.json', async () => { 99 | const output = await exec( 100 | 'node', 101 | './bin/cml.js', 102 | 'publish', 103 | '--mime-type', 104 | 'application/json', 105 | 'assets/vega-lite.json' 106 | ); 107 | 108 | expect(output.startsWith('https://')).toBe(true); 109 | expect(output.includes('cml=json')).toBe(true); 110 | }); 111 | 112 | test('cml publish assets/test.svg in Gitlab storage', async () => { 113 | const { TEST_GITLAB_REPOSITORY: repo, TEST_GITLAB_TOKEN: token } = 114 | process.env; 115 | 116 | const output = await exec( 117 | 'node', 118 | './bin/cml.js', 119 | 'publish', 120 | '--repo', 121 | repo, 122 | '--token', 123 | token, 124 | '--gitlab-uploads', 125 | 'assets/test.svg' 126 | ); 127 | 128 | expect(output.startsWith('/uploads/')).toBe(true); 129 | }); 130 | 131 | test('cml publish /nonexistent produces file error', async () => { 132 | await expect( 133 | exec('node', './bin/cml.js', 'publish', '/nonexistent') 134 | ).rejects.toThrowError('ENOENT'); 135 | }); 136 | }); 137 | -------------------------------------------------------------------------------- /bin/cml/asset/publish.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs').promises; 2 | const kebabcaseKeys = require('kebabcase-keys'); 3 | const { logger } = require('../../../src/logger'); 4 | 5 | const { CML } = require('../../../src/cml'); 6 | 7 | const DESCRIPTION = 'Publish an asset'; 8 | const DOCSURL = 'https://cml.dev/doc/usage#cml-reports'; 9 | 10 | exports.command = 'publish '; 11 | exports.description = `${DESCRIPTION}\n${DOCSURL}`; 12 | 13 | exports.handler = async (opts) => { 14 | if (opts.gitlabUploads) { 15 | logger.warn( 16 | '--gitlab-uploads will be deprecated soon, use --native instead' 17 | ); 18 | opts.native = true; 19 | } 20 | 21 | const { file, asset: path } = opts; 22 | const cml = new CML({ ...opts }); 23 | const output = await cml.publish({ ...opts, path }); 24 | 25 | if (!file) console.log(output); 26 | else await fs.writeFile(file, output); 27 | }; 28 | 29 | exports.builder = (yargs) => 30 | yargs 31 | .env('CML_ASSET') 32 | .option('options', { default: exports.options, hidden: true }) 33 | .options(exports.options); 34 | 35 | exports.options = kebabcaseKeys({ 36 | url: { 37 | type: 'string', 38 | description: 'Self-Hosted URL', 39 | hidden: true 40 | }, 41 | md: { 42 | type: 'boolean', 43 | description: 'Output in markdown format [title || name](url)' 44 | }, 45 | title: { 46 | type: 'string', 47 | alias: 't', 48 | description: 'Markdown title [title](url) or ![](url title)' 49 | }, 50 | native: { 51 | type: 'boolean', 52 | description: 53 | "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub" 54 | }, 55 | gitlabUploads: { 56 | type: 'boolean', 57 | hidden: true 58 | }, 59 | rmWatermark: { 60 | type: 'boolean', 61 | description: 'Avoid CML watermark.', 62 | hidden: true, 63 | telemetryData: 'name' 64 | }, 65 | mimeType: { 66 | type: 'string', 67 | defaultDescription: 'infer from the file contents', 68 | description: 'MIME type' 69 | }, 70 | file: { 71 | type: 'string', 72 | alias: 'f', 73 | description: 74 | 'Append the output to the given file or create it if does not exist', 75 | hidden: true 76 | }, 77 | repo: { 78 | type: 'string', 79 | description: 80 | 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' 81 | } 82 | }); 83 | exports.DOCSURL = DOCSURL; 84 | -------------------------------------------------------------------------------- /bin/cml/asset/publish.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | 3 | describe('CML cli test', () => { 4 | test('cml publish --help', async () => { 5 | const output = await exec('node', './bin/cml.js', 'publish', '--help'); 6 | 7 | expect(output).toMatchInlineSnapshot(` 8 | "cml.js publish 9 | 10 | Global Options: 11 | --log Logging verbosity 12 | [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] 13 | --driver Git provider where the repository is hosted 14 | [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the 15 | environment] 16 | --repo Specifies the repo to be used. If not specified 17 | is extracted from the CI ENV. 18 | [string] [default: infer from the environment] 19 | --driver-token, --token CI driver personal/project access token (PAT) 20 | [string] [default: infer from the environment] 21 | --help Show help [boolean] 22 | 23 | Options: 24 | --md Output in markdown format [title || name](url) [boolean] 25 | -t, --title Markdown title [title](url) or ![](url title) [string] 26 | --native Uses driver's native capabilities to upload assets instead of 27 | CML's storage; not available on GitHub [boolean] 28 | --mime-type MIME type [string] [default: infer from the file contents]" 29 | `); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /bin/cml/check.js: -------------------------------------------------------------------------------- 1 | exports.command = 'check'; 2 | exports.description = 'Manage CI checks'; 3 | exports.builder = (yargs) => 4 | yargs 5 | .commandDir('./check', { exclude: /\.test\.js$/ }) 6 | .recommendCommands() 7 | .demandCommand() 8 | .strict(); 9 | -------------------------------------------------------------------------------- /bin/cml/check/create.e2e.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | const fs = require('fs').promises; 3 | 4 | describe('CML e2e', () => { 5 | const path = 'check.md'; 6 | 7 | afterEach(async () => { 8 | try { 9 | await fs.unlink(path); 10 | } catch (err) {} 11 | }); 12 | 13 | test('cml send-github-check', async () => { 14 | const report = `## Test Check Report`; 15 | 16 | await fs.writeFile(path, report); 17 | process.env.GITHUB_ACTIONS && 18 | (await exec('node', './bin/cml.js', 'send-github-check', path)); 19 | }); 20 | 21 | test('cml send-github-check failure with tile "CML neutral test"', async () => { 22 | const report = `## Hi this check should be neutral`; 23 | const title = 'CML neutral test'; 24 | const conclusion = 'neutral'; 25 | 26 | await fs.writeFile(path, report); 27 | process.env.GITHUB_ACTIONS && 28 | (await exec( 29 | 'node', 30 | './bin/cml.js', 31 | 'send-github-check', 32 | path, 33 | '--title', 34 | title, 35 | '--conclusion', 36 | conclusion 37 | )); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /bin/cml/check/create.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs').promises; 2 | const kebabcaseKeys = require('kebabcase-keys'); 3 | 4 | const DESCRIPTION = 'Create a check report'; 5 | const DOCSURL = 'https://cml.dev/doc/ref/check'; 6 | 7 | exports.command = 'create '; 8 | exports.description = `${DESCRIPTION}\n${DOCSURL}`; 9 | 10 | exports.handler = async (opts) => { 11 | const { cml, markdownfile } = opts; 12 | const report = await fs.readFile(markdownfile, 'utf-8'); 13 | await cml.checkCreate({ ...opts, report }); 14 | }; 15 | 16 | exports.builder = (yargs) => 17 | yargs 18 | .env('CML_CHECK') 19 | .option('options', { default: exports.options, hidden: true }) 20 | .options(exports.options); 21 | 22 | exports.options = kebabcaseKeys({ 23 | token: { 24 | type: 'string', 25 | description: 26 | "GITHUB_TOKEN or Github App token. Personal access token won't work" 27 | }, 28 | commitSha: { 29 | type: 'string', 30 | alias: 'head-sha', 31 | defaultDescription: 'HEAD', 32 | description: 'Commit SHA linked to this comment' 33 | }, 34 | conclusion: { 35 | type: 'string', 36 | choices: [ 37 | 'success', 38 | 'failure', 39 | 'neutral', 40 | 'cancelled', 41 | 'skipped', 42 | 'timed_out' 43 | ], 44 | default: 'success', 45 | description: 'Conclusion status of the check' 46 | }, 47 | status: { 48 | type: 'string', 49 | choices: ['queued', 'in_progress', 'completed'], 50 | default: 'completed', 51 | description: 'Status of the check' 52 | }, 53 | title: { 54 | type: 'string', 55 | default: 'CML Report', 56 | description: 'Title of the check' 57 | } 58 | }); 59 | exports.DOCSURL = DOCSURL; 60 | -------------------------------------------------------------------------------- /bin/cml/check/create.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | 3 | describe('CML e2e', () => { 4 | test('cml send-github-check --help', async () => { 5 | const output = await exec( 6 | 'node', 7 | './bin/cml.js', 8 | 'send-github-check', 9 | '--help' 10 | ); 11 | 12 | expect(output).toMatchInlineSnapshot(` 13 | "cml.js send-github-check 14 | 15 | Global Options: 16 | --log Logging verbosity 17 | [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] 18 | --driver Git provider where the repository is hosted 19 | [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the 20 | environment] 21 | --repo Repository URL or slug 22 | [string] [default: infer from the environment] 23 | --driver-token, --token GITHUB_TOKEN or Github App token. Personal access 24 | token won't work 25 | [string] [default: infer from the environment] 26 | --help Show help [boolean] 27 | 28 | Options: 29 | --commit-sha, --head-sha Commit SHA linked to this comment 30 | [string] [default: HEAD] 31 | --conclusion Conclusion status of the check 32 | [string] [choices: \\"success\\", \\"failure\\", \\"neutral\\", \\"cancelled\\", \\"skipped\\", 33 | \\"timed_out\\"] [default: \\"success\\"] 34 | --status Status of the check 35 | [string] [choices: \\"queued\\", \\"in_progress\\", \\"completed\\"] [default: 36 | \\"completed\\"] 37 | --title Title of the check [string] [default: \\"CML Report\\"]" 38 | `); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /bin/cml/comment.js: -------------------------------------------------------------------------------- 1 | exports.command = 'comment'; 2 | exports.description = 'Manage comments'; 3 | exports.builder = (yargs) => 4 | yargs 5 | .commandDir('./comment', { exclude: /\.test\.js$/ }) 6 | .recommendCommands() 7 | .demandCommand() 8 | .strict(); 9 | -------------------------------------------------------------------------------- /bin/cml/comment/create.e2e.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | const fs = require('fs').promises; 3 | 4 | describe('Comment integration tests', () => { 5 | const path = 'comment.md'; 6 | 7 | afterEach(async () => { 8 | try { 9 | await fs.unlink(path); 10 | } catch (err) {} 11 | }); 12 | 13 | test('cml send-comment to specific repo', async () => { 14 | const { 15 | TEST_GITHUB_REPOSITORY: repo, 16 | TEST_GITHUB_TOKEN: token, 17 | TEST_GITHUB_COMMIT: sha 18 | } = process.env; 19 | 20 | const report = `## Test Comment Report specific`; 21 | 22 | await fs.writeFile(path, report); 23 | await exec( 24 | 'node', 25 | './bin/cml.js', 26 | 'send-comment', 27 | '--repo', 28 | repo, 29 | '--token', 30 | token, 31 | '--commit-sha', 32 | sha, 33 | path 34 | ); 35 | }); 36 | }); 37 | -------------------------------------------------------------------------------- /bin/cml/comment/create.js: -------------------------------------------------------------------------------- 1 | const kebabcaseKeys = require('kebabcase-keys'); 2 | 3 | const DESCRIPTION = 'Create a comment'; 4 | const DOCSURL = 'https://cml.dev/doc/ref/comment#create'; 5 | 6 | exports.command = 'create '; 7 | exports.description = `${DESCRIPTION}\n${DOCSURL}`; 8 | 9 | exports.handler = async (opts) => { 10 | const { cml } = opts; 11 | console.log(await cml.commentCreate(opts)); 12 | }; 13 | 14 | exports.builder = (yargs) => 15 | yargs 16 | .env('CML_COMMENT') 17 | .option('options', { default: exports.options, hidden: true }) 18 | .options(exports.options); 19 | 20 | exports.options = kebabcaseKeys({ 21 | target: { 22 | type: 'string', 23 | description: 24 | 'Comment type (`commit`, `pr`, `commit/f00bar`, `pr/42`, `issue/1337`),' + 25 | 'default is automatic (`pr` but fallback to `commit`).' 26 | }, 27 | pr: { 28 | type: 'boolean', 29 | description: 30 | 'Post to an existing PR/MR associated with the specified commit', 31 | conflicts: ['target', 'commitSha'], 32 | hidden: true 33 | }, 34 | commitSha: { 35 | type: 'string', 36 | alias: 'head-sha', 37 | description: 'Commit SHA linked to this comment', 38 | conflicts: ['target', 'pr'], 39 | hidden: true 40 | }, 41 | watch: { 42 | type: 'boolean', 43 | description: 'Watch for changes and automatically update the comment' 44 | }, 45 | triggerFile: { 46 | type: 'string', 47 | description: 'File used to trigger the watcher', 48 | hidden: true 49 | }, 50 | publish: { 51 | type: 'boolean', 52 | default: true, 53 | description: 'Upload any local images found in the Markdown report' 54 | }, 55 | publishUrl: { 56 | type: 'string', 57 | default: 'https://asset.cml.dev', 58 | description: 'Self-hosted image server URL', 59 | telemetryData: 'name' 60 | }, 61 | publishNative: { 62 | type: 'boolean', 63 | alias: 'native', 64 | description: 65 | "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub", 66 | telemetryData: 'name' 67 | }, 68 | update: { 69 | type: 'boolean', 70 | description: 71 | 'Update the last CML comment (if any) instead of creating a new one', 72 | hidden: true 73 | }, 74 | rmWatermark: { 75 | type: 'boolean', 76 | description: 77 | 'Avoid watermark; CML needs a watermark to be able to distinguish CML comments from others', 78 | hidden: true, 79 | telemetryData: 'name' 80 | }, 81 | watermarkTitle: { 82 | type: 'string', 83 | description: 84 | 'Hidden comment marker (used for targeting in subsequent `cml comment update`); "{workflow}" & "{run}" are auto-replaced', 85 | default: '', 86 | conflicts: ['rmWatermark'] 87 | } 88 | }); 89 | exports.DOCSURL = DOCSURL; 90 | -------------------------------------------------------------------------------- /bin/cml/comment/create.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | 3 | describe('Comment integration tests', () => { 4 | test('cml send-comment --help', async () => { 5 | const output = await exec('node', './bin/cml.js', 'send-comment', '--help'); 6 | expect(output).toMatchInlineSnapshot(` 7 | "cml.js send-comment 8 | 9 | Global Options: 10 | --log Logging verbosity 11 | [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] 12 | --driver Git provider where the repository is hosted 13 | [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the 14 | environment] 15 | --repo Repository URL or slug 16 | [string] [default: infer from the environment] 17 | --driver-token, --token CI driver personal/project access token (PAT) 18 | [string] [default: infer from the environment] 19 | --help Show help [boolean] 20 | 21 | Options: 22 | --target Comment type (\`commit\`, \`pr\`, \`commit/f00bar\`, 23 | \`pr/42\`, \`issue/1337\`),default is automatic (\`pr\` 24 | but fallback to \`commit\`). [string] 25 | --watch Watch for changes and automatically update the 26 | comment [boolean] 27 | --publish Upload any local images found in the Markdown 28 | report [boolean] [default: true] 29 | --publish-url Self-hosted image server URL 30 | [string] [default: \\"https://asset.cml.dev\\"] 31 | --publish-native, --native Uses driver's native capabilities to upload assets 32 | instead of CML's storage; not available on GitHub 33 | [boolean] 34 | --watermark-title Hidden comment marker (used for targeting in 35 | subsequent \`cml comment update\`); \\"{workflow}\\" & 36 | \\"{run}\\" are auto-replaced [string] [default: \\"\\"]" 37 | `); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /bin/cml/comment/update.js: -------------------------------------------------------------------------------- 1 | const { builder, handler } = require('./create'); 2 | 3 | const DESCRIPTION = 'Update a comment'; 4 | const DOCSURL = 'https://cml.dev/doc/ref/comment#update'; 5 | 6 | exports.command = 'update '; 7 | exports.description = `${DESCRIPTION}\n${DOCSURL}`; 8 | 9 | exports.handler = async (opts) => { 10 | await handler({ ...opts, update: true }); 11 | }; 12 | 13 | exports.builder = builder; 14 | exports.DOCSURL = DOCSURL; 15 | -------------------------------------------------------------------------------- /bin/cml/pr.js: -------------------------------------------------------------------------------- 1 | const { options, handler } = require('./pr/create'); 2 | 3 | exports.command = 'pr '; 4 | exports.description = 'Manage pull requests'; 5 | exports.handler = handler; 6 | exports.builder = (yargs) => 7 | yargs 8 | .commandDir('./pr', { exclude: /\.test\.js$/ }) 9 | .recommendCommands() 10 | .env('CML_PR') 11 | .options( 12 | Object.fromEntries( 13 | Object.entries(options).map(([key, value]) => [ 14 | key, 15 | { ...value, hidden: true, global: false } 16 | ]) 17 | ) 18 | ) 19 | .option('options', { default: options, hidden: true }) 20 | .strict(); 21 | -------------------------------------------------------------------------------- /bin/cml/pr/create.e2e.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | 3 | describe('CML e2e', () => { 4 | test('cml-pr --help', async () => { 5 | const output = await exec('node', './bin/cml.js', 'pr', '--help'); 6 | 7 | expect(output).toMatchInlineSnapshot(` 8 | "cml.js pr 9 | 10 | Manage pull requests 11 | 12 | Commands: 13 | cml.js pr create [glob path...] Create a pull request (committing any given 14 | paths first) 15 | https://cml.dev/doc/ref/pr 16 | 17 | Global Options: 18 | --log Logging verbosity 19 | [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] 20 | --driver Git provider where the repository is hosted 21 | [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the 22 | environment] 23 | --repo Repository URL or slug 24 | [string] [default: infer from the environment] 25 | --driver-token, --token CI driver personal/project access token (PAT) 26 | [string] [default: infer from the environment] 27 | --help Show help [boolean]" 28 | `); 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /bin/cml/pr/create.js: -------------------------------------------------------------------------------- 1 | const kebabcaseKeys = require('kebabcase-keys'); 2 | 3 | const { 4 | GIT_REMOTE, 5 | GIT_USER_NAME, 6 | GIT_USER_EMAIL 7 | } = require('../../../src/cml'); 8 | 9 | const DESCRIPTION = 'Create a pull request (committing any given paths first)'; 10 | 11 | const DOCSURL = 'https://cml.dev/doc/ref/pr'; 12 | 13 | exports.command = 'create [glob path...]'; 14 | exports.description = `${DESCRIPTION}\n${DOCSURL}`; 15 | 16 | exports.handler = async (opts) => { 17 | const { cml, globpath: globs } = opts; 18 | const link = await cml.prCreate({ ...opts, globs }); 19 | console.log(link); 20 | }; 21 | 22 | exports.builder = (yargs) => 23 | yargs 24 | .env('CML_PR') 25 | .option('options', { default: exports.options, hidden: true }) 26 | .options(exports.options); 27 | 28 | exports.options = kebabcaseKeys({ 29 | md: { 30 | type: 'boolean', 31 | description: 'Output in markdown format [](url)' 32 | }, 33 | skipCI: { 34 | type: 'boolean', 35 | description: 'Force skip CI for the created commit (if any)' 36 | }, 37 | merge: { 38 | type: 'boolean', 39 | alias: 'auto-merge', 40 | conflicts: ['rebase', 'squash'], 41 | description: 'Try to merge the pull request upon creation' 42 | }, 43 | rebase: { 44 | type: 'boolean', 45 | conflicts: ['merge', 'squash'], 46 | description: 'Try to rebase-merge the pull request upon creation' 47 | }, 48 | squash: { 49 | type: 'boolean', 50 | conflicts: ['merge', 'rebase'], 51 | description: 'Try to squash-merge the pull request upon creation' 52 | }, 53 | branch: { 54 | type: 'string', 55 | description: 'Pull request branch name' 56 | }, 57 | targetBranch: { 58 | type: 'string', 59 | description: 'Pull request target branch name' 60 | }, 61 | title: { 62 | type: 'string', 63 | description: 'Pull request title' 64 | }, 65 | body: { 66 | type: 'string', 67 | description: 'Pull request description' 68 | }, 69 | message: { 70 | type: 'string', 71 | description: 'Commit message' 72 | }, 73 | remote: { 74 | type: 'string', 75 | default: GIT_REMOTE, 76 | description: 'Git remote' 77 | }, 78 | userEmail: { 79 | type: 'string', 80 | default: GIT_USER_EMAIL, 81 | description: 'Git user email' 82 | }, 83 | userName: { 84 | type: 'string', 85 | default: GIT_USER_NAME, 86 | description: 'Git user name' 87 | } 88 | }); 89 | exports.DOCSURL = DOCSURL; 90 | -------------------------------------------------------------------------------- /bin/cml/repo.js: -------------------------------------------------------------------------------- 1 | exports.command = 'repo'; 2 | exports.description = false; 3 | exports.builder = (yargs) => 4 | yargs 5 | .commandDir('./repo', { exclude: /\.test\.js$/ }) 6 | .recommendCommands() 7 | .demandCommand() 8 | .strict(); 9 | -------------------------------------------------------------------------------- /bin/cml/repo/prepare.js: -------------------------------------------------------------------------------- 1 | const kebabcaseKeys = require('kebabcase-keys'); 2 | 3 | const { GIT_USER_NAME, GIT_USER_EMAIL } = require('../../../src/cml'); 4 | 5 | const DESCRIPTION = 'Prepare the cloned repository'; 6 | const DOCSURL = 'https://cml.dev/doc/ref/ci'; 7 | 8 | exports.command = 'prepare'; 9 | exports.description = `${DESCRIPTION}\n${DOCSURL}`; 10 | 11 | exports.handler = async (opts) => { 12 | const { cml } = opts; 13 | await cml.ci(opts); 14 | }; 15 | 16 | exports.builder = (yargs) => 17 | yargs 18 | .env('CML_REPO') 19 | .option('options', { default: exports.options, hidden: true }) 20 | .options(exports.options); 21 | 22 | exports.options = kebabcaseKeys({ 23 | fetchDepth: { 24 | type: 'number', 25 | description: 'Number of commits to fetch (use `0` for all branches & tags)' 26 | }, 27 | unshallow: { 28 | type: 'boolean', 29 | description: 30 | 'Fetch as much as possible, converting a shallow repository to a complete one', 31 | hidden: true 32 | }, 33 | userEmail: { 34 | type: 'string', 35 | default: GIT_USER_EMAIL, 36 | description: 'Git user email' 37 | }, 38 | userName: { 39 | type: 'string', 40 | default: GIT_USER_NAME, 41 | description: 'Git user name' 42 | } 43 | }); 44 | exports.DOCSURL = DOCSURL; 45 | -------------------------------------------------------------------------------- /bin/cml/repo/prepare.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | 3 | describe('CML e2e', () => { 4 | test('cml-ci --help', async () => { 5 | const output = await exec('node', './bin/cml.js', 'ci', '--help'); 6 | 7 | expect(output).toMatchInlineSnapshot(` 8 | "cml.js ci 9 | 10 | Prepare Git repository for CML operations 11 | 12 | Global Options: 13 | --log Logging verbosity 14 | [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] 15 | --driver Git provider where the repository is hosted 16 | [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the 17 | environment] 18 | --repo Repository URL or slug 19 | [string] [default: infer from the environment] 20 | --driver-token, --token CI driver personal/project access token (PAT) 21 | [string] [default: infer from the environment] 22 | --help Show help [boolean] 23 | 24 | Options: 25 | --fetch-depth Number of commits to fetch (use \`0\` for all branches & tags) 26 | [number] 27 | --user-email Git user email [string] [default: \\"olivaw@iterative.ai\\"] 28 | --user-name Git user name [string] [default: \\"Olivaw[bot]\\"]" 29 | `); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /bin/cml/runner.js: -------------------------------------------------------------------------------- 1 | const { options, handler } = require('./runner/launch'); 2 | 3 | exports.command = 'runner'; 4 | exports.description = 'Manage self-hosted (cloud & on-premise) CI runners'; 5 | exports.handler = handler; 6 | exports.builder = (yargs) => 7 | yargs 8 | .commandDir('./runner', { exclude: /\.test\.js$/ }) 9 | .recommendCommands() 10 | .env('CML_RUNNER') 11 | .options( 12 | Object.fromEntries( 13 | Object.entries(options).map(([key, value]) => [ 14 | key, 15 | { ...value, hidden: true, global: false } 16 | ]) 17 | ) 18 | ) 19 | .option('options', { default: options, hidden: true }) 20 | .check(() => process.argv.some((arg) => arg.startsWith('-'))) 21 | .strict(); 22 | -------------------------------------------------------------------------------- /bin/cml/runner/launch.e2e.test.js: -------------------------------------------------------------------------------- 1 | jest.setTimeout(2000000); 2 | 3 | const isIp = require('is-ip'); 4 | const { CML } = require('../../../src/cml'); 5 | const { exec, sshConnection, randid, sleep } = require('../../../src/utils'); 6 | 7 | const IDLE_TIMEOUT = 15; 8 | const { 9 | TEST_GITHUB_TOKEN, 10 | TEST_GITHUB_REPOSITORY, 11 | TEST_GITLAB_TOKEN, 12 | TEST_GITLAB_REPOSITORY, 13 | SSH_PRIVATE 14 | } = process.env; 15 | 16 | const launchRunner = async (opts) => { 17 | const { cloud, type, repo, token, privateKey, name } = opts; 18 | const output = await exec( 19 | 'node', 20 | './bin/cml.js', 21 | 'runner', 22 | '--cloud', 23 | cloud, 24 | '--cloud-type', 25 | type, 26 | '--repo', 27 | repo, 28 | '--token', 29 | token, 30 | '--cloud-ssh-private', 31 | privateKey, 32 | '--name', 33 | name, 34 | '--cloud-spot', 35 | 'true', 36 | '--idle-timeout', 37 | IDLE_TIMEOUT 38 | ); 39 | const state = JSON.parse(output.split(/\n/).pop()); 40 | 41 | return state; 42 | }; 43 | 44 | const testRunner = async (opts) => { 45 | const { repo, token, name, privateKey } = opts; 46 | const { instanceIp: host } = await launchRunner(opts); 47 | expect(isIp(host)).toBe(true); 48 | 49 | const sshOpts = { host, username: 'ubuntu', privateKey }; 50 | const cml = new CML({ repo, token }); 51 | 52 | let runner = await cml.runnerByName({ name }); 53 | expect(runner).not.toBe(undefined); 54 | await sshConnection(sshOpts); 55 | 56 | await sleep(IDLE_TIMEOUT + 60); 57 | 58 | runner = await cml.runnerByName({ name }); 59 | expect(runner).toBe(undefined); 60 | 61 | let sshErr; 62 | try { 63 | await sshConnection(sshOpts); 64 | } catch (err) { 65 | sshErr = err; 66 | } 67 | expect(sshErr).not.toBe(undefined); 68 | }; 69 | 70 | describe('CML e2e', () => { 71 | test.skip('cml-runner GL/AWS', async () => { 72 | const opts = { 73 | repo: TEST_GITLAB_REPOSITORY, 74 | token: TEST_GITLAB_TOKEN, 75 | privateKey: SSH_PRIVATE, 76 | cloud: 'aws', 77 | type: 't2.micro', 78 | name: `cml-test-${randid()}` 79 | }; 80 | 81 | await testRunner(opts); 82 | }); 83 | 84 | test.skip('cml-runner GH/AWS', async () => { 85 | const opts = { 86 | repo: TEST_GITHUB_REPOSITORY, 87 | token: TEST_GITHUB_TOKEN, 88 | privateKey: SSH_PRIVATE, 89 | cloud: 'aws', 90 | type: 't2.micro', 91 | name: `cml-test-${randid()}` 92 | }; 93 | 94 | await testRunner(opts); 95 | }); 96 | 97 | test.skip('cml-runner GL/Azure', async () => { 98 | const opts = { 99 | repo: TEST_GITLAB_REPOSITORY, 100 | token: TEST_GITLAB_TOKEN, 101 | privateKey: SSH_PRIVATE, 102 | cloud: 'azure', 103 | type: 'm', 104 | name: `cml-test-${randid()}` 105 | }; 106 | 107 | await testRunner(opts); 108 | }); 109 | 110 | test.skip('cml-runner GH/Azure', async () => { 111 | const opts = { 112 | repo: TEST_GITHUB_REPOSITORY, 113 | token: TEST_GITHUB_TOKEN, 114 | privateKey: SSH_PRIVATE, 115 | cloud: 'azure', 116 | type: 'm', 117 | name: `cml-test-${randid()}` 118 | }; 119 | 120 | await testRunner(opts); 121 | }); 122 | 123 | test.skip('cml-runner GL/GCP', async () => { 124 | const opts = { 125 | repo: TEST_GITLAB_REPOSITORY, 126 | token: TEST_GITLAB_TOKEN, 127 | privateKey: SSH_PRIVATE, 128 | cloud: 'gcp', 129 | type: 'm', 130 | name: `cml-test-${randid()}` 131 | }; 132 | 133 | await testRunner(opts); 134 | }); 135 | 136 | test.skip('cml-runner GH/GCP', async () => { 137 | const opts = { 138 | repo: TEST_GITHUB_REPOSITORY, 139 | token: TEST_GITHUB_TOKEN, 140 | privateKey: SSH_PRIVATE, 141 | cloud: 'gcp', 142 | type: 'm', 143 | name: `cml-test-${randid()}` 144 | }; 145 | 146 | await testRunner(opts); 147 | }); 148 | }); 149 | -------------------------------------------------------------------------------- /bin/cml/runner/launch.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | 3 | describe('CML e2e', () => { 4 | test('cml-runner --help', async () => { 5 | const output = await exec('node', './bin/cml.js', 'runner', '--help'); 6 | 7 | expect(output).toMatchInlineSnapshot(` 8 | "cml.js runner 9 | 10 | Manage self-hosted (cloud & on-premise) CI runners 11 | 12 | Commands: 13 | cml.js runner launch Launch and register a self-hosted runner 14 | https://cml.dev/doc/ref/runner 15 | 16 | Global Options: 17 | --log Logging verbosity 18 | [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] 19 | --driver Git provider where the repository is hosted 20 | [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the 21 | environment] 22 | --repo Repository URL or slug 23 | [string] [default: infer from the environment] 24 | --driver-token, --token CI driver personal/project access token (PAT) 25 | [string] [default: infer from the environment] 26 | --help Show help [boolean]" 27 | `); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /bin/cml/tensorboard.js: -------------------------------------------------------------------------------- 1 | exports.command = 'tensorboard'; 2 | exports.description = false; 3 | exports.builder = (yargs) => 4 | yargs 5 | .options({ 6 | driver: { hidden: true }, 7 | repo: { hidden: true }, 8 | token: { hidden: true } 9 | }) 10 | .commandDir('./tensorboard', { exclude: /\.test\.js$/ }) 11 | .recommendCommands() 12 | .demandCommand() 13 | .strict(); 14 | -------------------------------------------------------------------------------- /bin/cml/tensorboard/connect.e2e.test.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs').promises; 2 | const tempy = require('tempy'); 3 | const { exec, isProcRunning, sleep } = require('../../../src/utils'); 4 | const { tbLink } = require('./connect'); 5 | 6 | const CREDENTIALS = 7 | '{"refresh_token": "1//03vy0UEEbtGrlCgYIARAAGAMSNwF-L9Irj4R63gOnR8ySz0CqG65smNVt7WWqV-fDGHqtzJr_3Vp71fbPnYgkpeGoDnIAVeP28c8", "token_uri": "https://oauth2.googleapis.com/token", "client_id": "373649185512-26ojik4u7dt0rdtfdmfnhpajqqh579qd.apps.googleusercontent.com", "client_secret": "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", "scopes": ["openid", "https://www.googleapis.com/auth/userinfo.email"], "type": "authorized_user"}'; 8 | 9 | const isTbRunning = async () => { 10 | await sleep(2); 11 | const isRunning = await isProcRunning({ name: 'tensorboard' }); 12 | 13 | return isRunning; 14 | }; 15 | 16 | const rmTbDevExperiment = async (tbOutput) => { 17 | const id = /experiment\/([a-zA-Z0-9]{22})/.exec(tbOutput)[1]; 18 | await exec('tensorboard', 'dev', 'delete', '--experiment_id', id); 19 | }; 20 | 21 | describe('tbLink', () => { 22 | test.skip('timeout without result throws exception', async () => { 23 | const stdout = tempy.file({ extension: 'log' }); 24 | const stderror = tempy.file({ extension: 'log' }); 25 | const message = 'there is an error'; 26 | let error; 27 | 28 | await fs.writeFile(stdout, 'nothing'); 29 | await fs.writeFile(stderror, message); 30 | 31 | try { 32 | await tbLink({ stdout, stderror, timeout: 5 }); 33 | } catch (err) { 34 | error = err; 35 | } 36 | 37 | expect(error.message).toBe(`Tensorboard took too long`); 38 | }); 39 | 40 | test.skip('valid url is returned', async () => { 41 | const stdout = tempy.file({ extension: 'log' }); 42 | const stderror = tempy.file({ extension: 'log' }); 43 | const message = 'https://iterative.ai'; 44 | 45 | await fs.writeFile(stdout, message); 46 | await fs.writeFile(stderror, ''); 47 | 48 | const link = await tbLink({ stderror, stdout, timeout: 5 }); 49 | expect(link).toBe(`${message}/?cml=tb`); 50 | }); 51 | }); 52 | 53 | describe('CML e2e', () => { 54 | test.skip('cml tensorboard-dev --md returns md and after command TB is still up', async () => { 55 | const name = 'My experiment'; 56 | const desc = 'Test experiment'; 57 | const title = 'go to the experiment'; 58 | const output = await exec( 59 | 'node', 60 | './bin/cml.js', 61 | 'tensorboard-dev', 62 | '--credentials', 63 | CREDENTIALS, 64 | '--md', 65 | '--title', 66 | title, 67 | '--logdir', 68 | 'logs', 69 | '--name', 70 | name, 71 | '--description', 72 | desc 73 | ); 74 | 75 | const isRunning = await isTbRunning(); 76 | await rmTbDevExperiment(output); 77 | 78 | expect(isRunning).toBe(true); 79 | expect(output.startsWith(`[${title}](https://`)).toBe(true); 80 | expect(output.includes('cml=tb')).toBe(true); 81 | }); 82 | 83 | test.skip('cml tensorboard-dev invalid creds', async () => { 84 | try { 85 | await exec( 86 | 'node', 87 | './bin/cml.js', 88 | 'tensorboard-dev', 89 | '--credentials', 90 | 'invalid' 91 | ); 92 | } catch (err) { 93 | expect(err.message.includes('json.decoder.JSONDecodeError')).toBe(true); 94 | } 95 | }); 96 | }); 97 | -------------------------------------------------------------------------------- /bin/cml/tensorboard/connect.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs').promises; 2 | const kebabcaseKeys = require('kebabcase-keys'); 3 | const { spawn } = require('child_process'); 4 | const { homedir } = require('os'); 5 | const tempy = require('tempy'); 6 | const { logger } = require('../../../src/logger'); 7 | 8 | const { exec, watermarkUri, sleep } = require('../../../src/utils'); 9 | const yargs = require('yargs'); 10 | 11 | const tbLink = async (opts = {}) => { 12 | const { stdout, stderror, title, name, rmWatermark, md, timeout = 60 } = opts; 13 | 14 | let chrono = 0; 15 | const chronoStep = 2; 16 | while (chrono < timeout) { 17 | const data = await fs.readFile(stdout, 'utf8'); 18 | const urls = data.match(/(https?:\/\/[^\s]+)/) || []; 19 | 20 | if (urls.length) { 21 | let [output] = urls; 22 | 23 | if (!rmWatermark) output = watermarkUri({ uri: output, type: 'tb' }); 24 | if (md) output = `[${title || name}](${output})`; 25 | 26 | return output; 27 | } 28 | 29 | await sleep(chronoStep); 30 | chrono = chrono + chronoStep; 31 | } 32 | 33 | logger.error(await fs.readFile(stderror, 'utf8')); 34 | throw new Error(`Tensorboard took too long`); 35 | }; 36 | 37 | const launchAndWaitLink = async (opts = {}) => { 38 | const { md, logdir, name, title, rmWatermark, extraParams } = opts; 39 | 40 | const command = `tensorboard dev upload --logdir ${logdir} ${extraParams}`; 41 | 42 | const stdoutPath = tempy.file({ extension: 'log' }); 43 | const stdoutFd = await fs.open(stdoutPath, 'a'); 44 | const stderrPath = tempy.file({ extension: 'log' }); 45 | const stderrFd = await fs.open(stderrPath, 'a'); 46 | 47 | const proc = spawn(command, [], { 48 | detached: true, 49 | shell: true, 50 | stdio: ['ignore', stdoutFd, stderrFd] 51 | }); 52 | 53 | proc.unref(); 54 | proc.on('exit', async (code, signal) => { 55 | if (code || signal) { 56 | logger.error(await fs.readFile(stderrPath, 'utf8')); 57 | throw new Error(`Tensorboard failed with error ${code || signal}`); 58 | } 59 | }); 60 | 61 | const url = await exports.tbLink({ 62 | stdout: stdoutPath, 63 | stderror: stderrPath, 64 | title, 65 | name, 66 | rmWatermark, 67 | md 68 | }); 69 | 70 | stdoutFd.close(); 71 | stderrFd.close(); 72 | 73 | return url; 74 | }; 75 | 76 | exports.tbLink = tbLink; 77 | 78 | const DESCRIPTION = 'Connect to tensorboard.dev and get a link'; 79 | const DOCSURL = 'https://cml.dev/doc/ref/tensorboard'; 80 | 81 | exports.command = 'connect'; 82 | exports.description = `${DESCRIPTION}\n${DOCSURL}`; 83 | 84 | exports.handler = async (opts) => { 85 | if (new Date() > new Date('2024-01-01')) { 86 | logger.error('TensorBoard.dev has been shut down as of January 1, 2024'); 87 | yargs.exit(1); 88 | } 89 | 90 | const { file, credentials, name, description } = opts; 91 | 92 | const path = `${homedir()}/.config/tensorboard/credentials`; 93 | await fs.mkdir(path, { recursive: true }); 94 | await fs.writeFile(`${path}/uploader-creds.json`, credentials); 95 | 96 | const help = await exec('tensorboard', 'dev', 'upload', '-h'); 97 | const extraParamsFound = 98 | (name || description) && help.indexOf('--description') >= 0; 99 | const extraParams = extraParamsFound 100 | ? `--name "${name}" --description "${description}"` 101 | : ''; 102 | 103 | const url = await launchAndWaitLink({ ...opts, extraParams }); 104 | if (!file) console.log(url); 105 | else await fs.appendFile(file, url); 106 | }; 107 | 108 | exports.builder = (yargs) => 109 | yargs 110 | .env('CML_TENSORBOARD') 111 | .option('options', { default: exports.options, hidden: true }) 112 | .options(exports.options); 113 | 114 | exports.options = kebabcaseKeys({ 115 | credentials: { 116 | type: 'string', 117 | alias: 'c', 118 | required: true, 119 | description: 120 | 'TensorBoard credentials as JSON, usually found at ~/.config/tensorboard/credentials/uploader-creds.json' 121 | }, 122 | logdir: { 123 | type: 'string', 124 | description: 'Directory containing the logs to process' 125 | }, 126 | name: { 127 | type: 'string', 128 | description: 'Tensorboard experiment title; max 100 characters' 129 | }, 130 | description: { 131 | type: 'string', 132 | description: 133 | 'Tensorboard experiment description in Markdown format; max 600 characters' 134 | }, 135 | md: { 136 | type: 'boolean', 137 | description: 'Output as markdown [title || name](url)' 138 | }, 139 | title: { 140 | type: 'string', 141 | alias: 't', 142 | description: 'Markdown title, if not specified, param name will be used' 143 | }, 144 | file: { 145 | type: 'string', 146 | alias: 'f', 147 | description: 148 | 'Append the output to the given file or create it if does not exist', 149 | hidden: true 150 | }, 151 | rmWatermark: { 152 | type: 'boolean', 153 | description: 'Avoid CML watermark', 154 | hidden: true, 155 | telemetryData: 'name' 156 | } 157 | }); 158 | exports.DOCSURL = DOCSURL; 159 | -------------------------------------------------------------------------------- /bin/cml/tensorboard/connect.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | 3 | describe('CML e2e', () => { 4 | test('cml tensorboard-dev --help', async () => { 5 | const output = await exec( 6 | 'node', 7 | './bin/cml.js', 8 | 'tensorboard-dev', 9 | '--help' 10 | ); 11 | 12 | expect(output).toMatchInlineSnapshot(` 13 | "cml.js tensorboard-dev 14 | 15 | Global Options: 16 | --log Logging verbosity 17 | [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] 18 | --driver Git provider where the repository is hosted 19 | [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the 20 | environment] 21 | --repo Repository URL or slug 22 | [string] [default: infer from the environment] 23 | --driver-token, --token CI driver personal/project access token (PAT) 24 | [string] [default: infer from the environment] 25 | --help Show help [boolean] 26 | 27 | Options: 28 | -c, --credentials TensorBoard credentials as JSON, usually found at 29 | ~/.config/tensorboard/credentials/uploader-creds.json 30 | [string] [required] 31 | --logdir Directory containing the logs to process [string] 32 | --name Tensorboard experiment title; max 100 characters [string] 33 | --description Tensorboard experiment description in Markdown format; max 34 | 600 characters [string] 35 | --md Output as markdown [title || name](url) [boolean] 36 | -t, --title Markdown title, if not specified, param name will be used 37 | [string]" 38 | `); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /bin/cml/workflow.js: -------------------------------------------------------------------------------- 1 | exports.command = 'workflow'; 2 | exports.description = 'Manage CI workflows'; 3 | exports.builder = (yargs) => 4 | yargs 5 | .commandDir('./workflow', { exclude: /\.test\.js$/ }) 6 | .recommendCommands() 7 | .demandCommand() 8 | .strict(); 9 | -------------------------------------------------------------------------------- /bin/cml/workflow/rerun.js: -------------------------------------------------------------------------------- 1 | const kebabcaseKeys = require('kebabcase-keys'); 2 | 3 | const DESCRIPTION = 'Rerun a workflow given the jobId or workflowId'; 4 | const DOCSURL = 'https://cml.dev/doc/ref/workflow'; 5 | 6 | exports.command = 'rerun'; 7 | exports.description = `${DESCRIPTION}\n${DOCSURL}`; 8 | 9 | exports.handler = async (opts) => { 10 | const { cml } = opts; 11 | await cml.pipelineRerun(opts); 12 | }; 13 | 14 | exports.builder = (yargs) => 15 | yargs 16 | .env('CML_WORKFLOW') 17 | .option('options', { default: exports.options, hidden: true }) 18 | .options(exports.options); 19 | 20 | exports.options = kebabcaseKeys({ 21 | id: { 22 | type: 'string', 23 | description: 'Run identifier to be rerun' 24 | } 25 | }); 26 | 27 | exports.DOCSURL = DOCSURL; 28 | -------------------------------------------------------------------------------- /bin/cml/workflow/rerun.test.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('../../../src/utils'); 2 | 3 | describe('CML e2e', () => { 4 | test('cml-ci --help', async () => { 5 | const output = await exec( 6 | 'node', 7 | './bin/cml.js', 8 | 'rerun-workflow', 9 | '--help' 10 | ); 11 | 12 | expect(output).toMatchInlineSnapshot(` 13 | "cml.js rerun-workflow 14 | 15 | Global Options: 16 | --log Logging verbosity 17 | [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] 18 | --driver Git provider where the repository is hosted 19 | [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the 20 | environment] 21 | --repo Repository URL or slug 22 | [string] [default: infer from the environment] 23 | --driver-token, --token CI driver personal/project access token (PAT) 24 | [string] [default: infer from the environment] 25 | --help Show help [boolean] 26 | 27 | Options: 28 | --id Run identifier to be rerun [string]" 29 | `); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /bin/legacy/commands/ci.js: -------------------------------------------------------------------------------- 1 | const { builder, handler } = require('../../cml/repo/prepare'); 2 | 3 | exports.command = 'ci'; 4 | exports.description = 'Prepare Git repository for CML operations'; 5 | exports.handler = handler; 6 | exports.builder = builder; 7 | -------------------------------------------------------------------------------- /bin/legacy/commands/publish.js: -------------------------------------------------------------------------------- 1 | const { addDeprecationNotice } = require('../deprecation'); 2 | const { builder, handler } = require('../../cml/asset/publish'); 3 | 4 | exports.command = 'publish '; 5 | exports.description = false; 6 | exports.handler = handler; 7 | exports.builder = addDeprecationNotice({ 8 | builder, 9 | notice: 10 | '"cml publish" is deprecated since "cml comment" now supports "![inline](./asset.png)"' 11 | }); 12 | -------------------------------------------------------------------------------- /bin/legacy/commands/rerun-workflow.js: -------------------------------------------------------------------------------- 1 | const { addDeprecationNotice } = require('../deprecation'); 2 | const { builder, handler } = require('../../cml/workflow/rerun'); 3 | 4 | exports.command = 'rerun-workflow'; 5 | exports.description = false; 6 | exports.handler = handler; 7 | exports.builder = addDeprecationNotice({ 8 | builder, 9 | notice: '"cml rerun-workflow" is deprecated, please use "cml workflow rerun"' 10 | }); 11 | -------------------------------------------------------------------------------- /bin/legacy/commands/send-comment.js: -------------------------------------------------------------------------------- 1 | const { addDeprecationNotice } = require('../deprecation'); 2 | const { builder, handler } = require('../../cml/comment/create'); 3 | 4 | exports.command = 'send-comment '; 5 | exports.description = false; 6 | exports.handler = handler; 7 | exports.builder = addDeprecationNotice({ 8 | builder, 9 | notice: '"cml send-comment" is deprecated, please use "cml comment create"' 10 | }); 11 | -------------------------------------------------------------------------------- /bin/legacy/commands/send-github-check.js: -------------------------------------------------------------------------------- 1 | const { addDeprecationNotice } = require('../deprecation'); 2 | const { builder, handler } = require('../../cml/check/create'); 3 | 4 | exports.command = 'send-github-check '; 5 | exports.description = false; 6 | exports.handler = handler; 7 | exports.builder = addDeprecationNotice({ 8 | builder, 9 | notice: '"cml send-github-check" is deprecated, please use "cml check create"' 10 | }); 11 | -------------------------------------------------------------------------------- /bin/legacy/commands/tensorboard-dev.js: -------------------------------------------------------------------------------- 1 | const { addDeprecationNotice } = require('../deprecation'); 2 | const { builder, handler } = require('../../cml/tensorboard/connect'); 3 | 4 | exports.command = 'tensorboard-dev'; 5 | exports.description = false; 6 | exports.handler = handler; 7 | exports.builder = addDeprecationNotice({ 8 | builder, 9 | notice: 10 | '"cml tensorboard-dev" is deprecated, please use "cml tensorboard connect"' 11 | }); 12 | -------------------------------------------------------------------------------- /bin/legacy/deprecation.js: -------------------------------------------------------------------------------- 1 | const { logger } = require('../../src/logger'); 2 | 3 | // addDeprecationNotice adds middleware to the yargs chain to display a deprecation notice. 4 | const addDeprecationNotice = (opts = {}) => { 5 | const { builder, notice } = opts; 6 | return (yargs) => 7 | builder(yargs).middleware([(opts) => deprecationNotice(opts, notice)]); 8 | }; 9 | 10 | const deprecationNotice = (opts, notice) => { 11 | const { cml } = opts; 12 | const driver = cml.getDriver(); 13 | if (driver.warn) { 14 | driver.warn(notice); 15 | } else { 16 | logger.warn(notice); 17 | } 18 | }; 19 | 20 | exports.addDeprecationNotice = addDeprecationNotice; 21 | -------------------------------------------------------------------------------- /bin/legacy/link.e2e.test.js: -------------------------------------------------------------------------------- 1 | const { bin } = require('../../package.json'); 2 | const { exec } = require('../../src/utils'); 3 | 4 | const commands = Object.keys(bin) 5 | .filter((command) => command.startsWith('cml-')) 6 | .map((command) => command.replace('cml-', '')); 7 | 8 | describe('command-line interface tests', () => { 9 | for (const command of commands) 10 | test(`legacy cml-${command} behaves as the new cml ${command}`, async () => { 11 | const legacyOutput = await exec(`cml-${command}`, '--help'); 12 | const newOutput = await exec('cml', command, '--help'); 13 | expect(legacyOutput).toBe(newOutput); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /bin/legacy/link.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // This file provides backwards compatibility with the legacy cml-commands 4 | // specified in package.json by acting as a single BusyBox-like entrypoint 5 | // that detects the name of the executed symbolic link and invokes in turn 6 | // the main command. E.g. cml-command should be redirected to cml command. 7 | 8 | const { basename } = require('path'); 9 | const { pseudoexec } = require('pseudoexec'); 10 | 11 | const [, file, ...parameters] = process.argv; 12 | const [, base, command] = basename(file).match(/^(\w+)-(.+)$/); 13 | 14 | (async () => process.exit(await pseudoexec(base, [command, ...parameters])))(); 15 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@dvcorg/cml", 3 | "version": "0.20.6", 4 | "description": "

", 5 | "author": { 6 | "name": "Iterative Inc", 7 | "url": "http://cml.dev" 8 | }, 9 | "homepage": "https://github.com/iterative/cml#readme", 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/iterative/cml" 13 | }, 14 | "bugs": { 15 | "url": "https://github.com/iterative/cml/issues" 16 | }, 17 | "keywords": [ 18 | "ci/cd", 19 | "ci", 20 | "cd", 21 | "continuous-integration", 22 | "continuous-delivery", 23 | "developer-tools", 24 | "data science", 25 | "machine learning", 26 | "deep learning", 27 | "cml", 28 | "dvc" 29 | ], 30 | "license": "Apache-2.0", 31 | "engines": { 32 | "node": ">=16.0.0" 33 | }, 34 | "eslintConfig": { 35 | "env": { 36 | "browser": true, 37 | "commonjs": true, 38 | "es6": true, 39 | "jest": true 40 | }, 41 | "extends": [ 42 | "standard", 43 | "prettier" 44 | ], 45 | "globals": { 46 | "Atomics": "readonly", 47 | "SharedArrayBuffer": "readonly" 48 | }, 49 | "parserOptions": { 50 | "ecmaVersion": 2020 51 | }, 52 | "ignorePatterns": [ 53 | "assets/", 54 | "dist/", 55 | "node_modules/" 56 | ], 57 | "rules": { 58 | "camelcase": [ 59 | 1, 60 | { 61 | "properties": "never" 62 | } 63 | ], 64 | "prettier/prettier": "error" 65 | }, 66 | "plugins": [ 67 | "prettier" 68 | ] 69 | }, 70 | "prettier": { 71 | "arrowParens": "always", 72 | "singleQuote": true, 73 | "trailingComma": "none", 74 | "printWidth": 80, 75 | "tabWidth": 2, 76 | "useTabs": false, 77 | "proseWrap": "always" 78 | }, 79 | "main": "index.js", 80 | "bin": { 81 | "cml": "bin/cml.js", 82 | "cml-send-github-check": "bin/legacy/link.js", 83 | "cml-send-comment": "bin/legacy/link.js", 84 | "cml-publish": "bin/legacy/link.js", 85 | "cml-tensorboard-dev": "bin/legacy/link.js", 86 | "cml-runner": "bin/legacy/link.js", 87 | "cml-pr": "bin/legacy/link.js" 88 | }, 89 | "scripts": { 90 | "lintfix": "eslint --fix ./ && prettier --write '**/*.{js,json,md}'", 91 | "lint": "eslint ./", 92 | "test": "jest --forceExit", 93 | "test:unit": "jest --testPathIgnorePatterns='e2e.test.js$' --forceExit", 94 | "test:e2e": "jest --testNamePattern='e2e.test.js$' --forceExit", 95 | "snapshot": "jest --updateSnapshot" 96 | }, 97 | "husky": { 98 | "hooks": { 99 | "pre-commit": "lint-staged" 100 | } 101 | }, 102 | "lint-staged": { 103 | "*.js": [ 104 | "eslint --fix", 105 | "prettier --write" 106 | ], 107 | "*.{json,md,yaml,yml}": [ 108 | "prettier --write" 109 | ] 110 | }, 111 | "resolution": { 112 | "colors": "1.4.0" 113 | }, 114 | "dependencies": { 115 | "@actions/core": "^1.10.0", 116 | "@actions/github": "^4.0.0", 117 | "@npcz/magic": "^1.3.12", 118 | "@octokit/core": "^3.5.1", 119 | "@octokit/graphql": "^4.8.0", 120 | "@octokit/plugin-throttling": "^3.5.2", 121 | "@octokit/rest": "18.0.0", 122 | "appdirs": "^1.1.0", 123 | "chokidar": "^3.5.3", 124 | "colors": "1.4.0", 125 | "exponential-backoff": "^3.1.1", 126 | "form-data": "^3.0.1", 127 | "fs-extra": "^9.1.0", 128 | "getos": "^3.2.1", 129 | "git-url-parse": "^13.1.0", 130 | "globby": "^11.0.4", 131 | "https-proxy-agent": "^5.0.1", 132 | "is-docker": "2.2.1", 133 | "js-base64": "^3.7.5", 134 | "kebabcase-keys": "^1.0.0", 135 | "node-fetch": "^2.6.12", 136 | "node-ssh": "^12.0.0", 137 | "os-name": "^5.0.1", 138 | "proxy-agent": "^6.3.0", 139 | "pseudoexec": "^0.2.0", 140 | "remark": "^13.0.0", 141 | "semver": "^7.6.0", 142 | "simple-git": "^3.16.0", 143 | "strip-ansi": "^6.0.1", 144 | "strip-url-auth": "^1.0.1", 145 | "tar": "^6.2.1", 146 | "tempy": "^0.6.0", 147 | "timestring": "^6.0.0", 148 | "unist-util-visit": "^2.0.3", 149 | "uuid": "^8.3.2", 150 | "which": "^2.0.2", 151 | "winston": "^3.10.0", 152 | "yargs": "^17.7.2" 153 | }, 154 | "devDependencies": { 155 | "eslint": "^8.1.0", 156 | "eslint-config-prettier": "^6.15.0", 157 | "eslint-config-standard": "^14.1.0", 158 | "eslint-plugin-import": "^2.24.2", 159 | "eslint-plugin-node": "^11.0.0", 160 | "eslint-plugin-prettier": "^3.4.1", 161 | "eslint-plugin-promise": "^4.3.1", 162 | "eslint-plugin-standard": "^4.1.0", 163 | "husky": "^8.0.3", 164 | "is-ip": "^3.1.0", 165 | "jest": "^27.2.1", 166 | "lint-staged": "^13.2.1", 167 | "prettier": "^2.4.1", 168 | "transparent-proxy": "^1.8.7" 169 | }, 170 | "jest": { 171 | "globalSetup": "./tests/setup.js", 172 | "globalTeardown": "./tests/teardown.js", 173 | "testTimeout": 600000, 174 | "collectCoverage": true 175 | }, 176 | "pkg": { 177 | "bin": "bin/cml.js", 178 | "assets": [ 179 | "**/*.js", 180 | "node_modules/@npcz/magic/dist/*.wasm", 181 | "node_modules/@npcz/magic/dist/magic.mgc", 182 | "assets/magic.mgc" 183 | ], 184 | "targets": [ 185 | "node16-alpine-arm64", 186 | "node16-alpine-x64", 187 | "node16-macos-arm64", 188 | "node16-macos-x64", 189 | "node16-linux-arm64", 190 | "node16-linux-x64", 191 | "node16-linuxstatic-arm64", 192 | "node16-linuxstatic-x64", 193 | "node16-win-arm64", 194 | "node16-win-x64" 195 | ], 196 | "outputPath": "build" 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /src/analytics.e2e.test.js: -------------------------------------------------------------------------------- 1 | const { send, jitsuEventPayload, isCI } = require('./analytics'); 2 | const CML = require('../src/cml').default; 3 | 4 | const { TEST_GITHUB_TOKEN: TOKEN, TEST_GITHUB_REPOSITORY: REPO } = process.env; 5 | 6 | describe('analytics tests', () => { 7 | test('userId', async () => { 8 | const cml = new CML({ repo: REPO, token: TOKEN }); 9 | const action = 'test'; 10 | const cloud = 'azure'; 11 | const container = 'cml'; 12 | const more = { one: 1, two: 2 }; 13 | const extra = { cloud, container, ...more }; 14 | const error = 'Ouch!'; 15 | const regex = /\d+\.\d+\.\d+/; 16 | 17 | const pl = await jitsuEventPayload({ action, error, extra, cml }); 18 | expect(pl.user_id.length).toBe(36); 19 | expect(pl.action).toBe(action); 20 | expect(pl.interface).toBe('cli'); 21 | expect(pl.tool_name).toBe('cml'); 22 | expect(regex.test(pl.tool_version)).toBe(true); 23 | expect(pl.tool_source).toBe(''); 24 | expect(pl.os_name.length > 0).toBe(true); 25 | expect(pl.os_version.length > 0).toBe(true); 26 | expect(pl.backend).toBe(cloud); 27 | expect(pl.error).toBe(error); 28 | expect(Object.keys(pl.extra).sort()).toEqual( 29 | ['ci', 'container'].concat(Object.keys(more)).sort() 30 | ); 31 | 32 | if (isCI()) { 33 | expect(pl.group_id.length).toBe(36); 34 | expect(pl.extra.ci.length > 0).toBe(true); 35 | } 36 | }); 37 | 38 | test('Send should never fail', async () => { 39 | let error = null; 40 | try { 41 | const cml = new CML({ repo: REPO, token: TOKEN }); 42 | const action = 'test'; 43 | const event = await jitsuEventPayload({ action, error, cml }); 44 | await send({ event, endpoint: 'https://notfound.cml.dev' }); 45 | } catch (err) { 46 | error = err; 47 | } 48 | 49 | expect(error).toBeNull(); 50 | }); 51 | }); 52 | -------------------------------------------------------------------------------- /src/analytics.js: -------------------------------------------------------------------------------- 1 | const os = require('os'); 2 | const path = require('path'); 3 | const fs = require('fs').promises; 4 | 5 | const fetch = require('node-fetch'); 6 | const { ProxyAgent } = require('proxy-agent'); 7 | const { promisify } = require('util'); 8 | const { scrypt } = require('crypto'); 9 | const { v4: uuidv4, v5: uuidv5, parse } = require('uuid'); 10 | const { userConfigDir } = require('appdirs'); 11 | const { logger } = require('./logger'); 12 | const isDocker = require('is-docker'); 13 | 14 | const { version: VERSION } = require('../package.json'); 15 | const { exec, fileExists, getos } = require('./utils'); 16 | 17 | const { 18 | ITERATIVE_ANALYTICS_ENDPOINT = 'https://telemetry.cml.dev/api/v1/s2s/event?ip_policy=strict', 19 | ITERATIVE_ANALYTICS_TOKEN = 's2s.jtyjusrpsww4k9b76rrjri.bl62fbzrb7nd9n6vn5bpqt', 20 | ITERATIVE_DO_NOT_TRACK, 21 | CODESPACES, 22 | GITHUB_SERVER_URL, 23 | GITHUB_REPOSITORY_OWNER, 24 | GITHUB_ACTOR, 25 | GITHUB_REPOSITORY, 26 | CI_SERVER_URL, 27 | CI_PROJECT_ROOT_NAMESPACE, 28 | GITLAB_USER_NAME, 29 | GITLAB_USER_LOGIN, 30 | GITLAB_USER_ID, 31 | BITBUCKET_WORKSPACE, 32 | BITBUCKET_STEP_TRIGGERER_UUID, 33 | TF_BUILD, 34 | CI 35 | } = process.env; 36 | 37 | const ID_DO_NOT_TRACK = 'do-not-track'; 38 | 39 | const deterministic = async (data) => { 40 | if (!data) 41 | throw new Error("data is not set, can't calculate a deterministic uuid"); 42 | 43 | const namespace = uuidv5('iterative.ai', uuidv5.DNS); 44 | const name = await promisify(scrypt)(data, parse(namespace), 8, { 45 | N: 1 << 16, 46 | r: 8, 47 | p: 1, 48 | maxmem: 128 * 1024 ** 2 49 | }); 50 | 51 | return uuidv5(name.toString('hex'), namespace); 52 | }; 53 | 54 | const guessCI = () => { 55 | if (GITHUB_SERVER_URL && !CODESPACES) return 'github'; 56 | if (CI_SERVER_URL) return 'gitlab'; 57 | if (BITBUCKET_WORKSPACE) return 'bitbucket'; 58 | if (TF_BUILD) return 'azure'; 59 | if (CI) return 'unknown'; 60 | return ''; 61 | }; 62 | 63 | const isCI = () => { 64 | const ci = guessCI(); 65 | return ci.length > 0; 66 | }; 67 | 68 | const groupId = async () => { 69 | if (!isCI()) return ''; 70 | 71 | const ci = guessCI(); 72 | let rawId = 'CI'; 73 | if (ci === 'github') { 74 | rawId = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY_OWNER}`; 75 | } else if (ci === 'gitlab') { 76 | rawId = `${CI_SERVER_URL}/${CI_PROJECT_ROOT_NAMESPACE}`; 77 | } else if (ci === 'bitbucket') { 78 | rawId = `https://bitbucket.com/${BITBUCKET_WORKSPACE}`; 79 | } 80 | 81 | return await deterministic(rawId); 82 | }; 83 | 84 | const userId = async ({ cml } = {}) => { 85 | try { 86 | if (isCI()) { 87 | let rawId; 88 | const ci = guessCI(); 89 | if (ci === 'github') { 90 | const { name, login, id } = await cml 91 | .getDriver() 92 | .user({ name: GITHUB_ACTOR }); 93 | rawId = `${name || ''} ${login} ${id}`; 94 | } else if (ci === 'gitlab') { 95 | rawId = `${GITLAB_USER_NAME} ${GITLAB_USER_LOGIN} ${GITLAB_USER_ID}`; 96 | } else if (ci === 'bitbucket') { 97 | rawId = BITBUCKET_STEP_TRIGGERER_UUID; 98 | } else { 99 | rawId = await exec('git', 'log', '-1', "--pretty=format:'%ae'"); 100 | } 101 | 102 | return await deterministic(rawId); 103 | } 104 | 105 | let id = uuidv4(); 106 | const oldPath = userConfigDir('dvc/user_id', 'iterative'); 107 | const newPath = userConfigDir('iterative/telemetry'); 108 | const readId = async ({ fpath }) => { 109 | const content = (await fs.readFile(fpath)).toString('utf-8'); 110 | try { 111 | const { user_id: id } = JSON.parse(content); 112 | return id; 113 | } catch (err) { 114 | return content; 115 | } 116 | }; 117 | 118 | const writeId = async ({ fpath, id }) => { 119 | await fs.mkdir(path.dirname(fpath), { recursive: true }); 120 | await fs.writeFile(fpath, JSON.stringify({ user_id: id })); 121 | }; 122 | 123 | if (await fileExists(newPath)) { 124 | id = await readId({ fpath: newPath }); 125 | } else { 126 | if (await fileExists(oldPath)) id = await readId({ fpath: oldPath }); 127 | 128 | await writeId({ fpath: newPath, id }); 129 | } 130 | 131 | if (!(await fileExists(oldPath)) && id !== ID_DO_NOT_TRACK) 132 | await writeId({ fpath: oldPath, id }); 133 | 134 | return id; 135 | } catch (err) { 136 | logger.debug(`userId failure: ${err.message}`); 137 | } 138 | }; 139 | 140 | const OS = () => { 141 | if (process.platform === 'win32') return 'windows'; 142 | if (process.platform === 'darwin') return 'macos'; 143 | 144 | return process.platform; 145 | }; 146 | 147 | const jitsuEventPayload = async ({ 148 | action = '', 149 | error, 150 | extra = {}, 151 | cml 152 | } = {}) => { 153 | try { 154 | const { cloud: backend = '', ...extraRest } = extra; 155 | 156 | const osname = OS(); 157 | let { release = os.release() } = await getos(); 158 | if (osname === 'windows') { 159 | const [major, minor, build] = release.split('.'); 160 | release = `${build}.${major}.${minor}-`; 161 | } 162 | 163 | return { 164 | user_id: await userId({ cml }), 165 | group_id: await groupId(), 166 | action, 167 | interface: 'cli', 168 | tool_name: 'cml', 169 | tool_version: VERSION, 170 | tool_source: '', 171 | os_name: osname, 172 | os_version: release, 173 | backend, 174 | error, 175 | extra: { 176 | ...extraRest, 177 | ci: guessCI(), 178 | container: 179 | process.env._CML_CONTAINER_IMAGE === 'true' ? 'cml' : isDocker() 180 | } 181 | }; 182 | } catch (err) { 183 | return {}; 184 | } 185 | }; 186 | 187 | const send = async ({ 188 | event, 189 | endpoint = ITERATIVE_ANALYTICS_ENDPOINT, 190 | token = ITERATIVE_ANALYTICS_TOKEN 191 | } = {}) => { 192 | try { 193 | if (ITERATIVE_DO_NOT_TRACK) return; 194 | if (!event.user_id || event.user_id === ID_DO_NOT_TRACK) return; 195 | 196 | // Exclude runs from GitHub Codespaces at Iterative 197 | if (GITHUB_REPOSITORY.startsWith('iterative/')) return; 198 | 199 | // Exclude continuous integration tests and internal projects from analytics 200 | if ( 201 | [ 202 | 'dc16cd76-71b7-5afa-bf11-e85e02ee1554', // deterministic("https://github.com/iterative") 203 | 'b0e229bf-2598-54b7-a3e0-81869cdad579', // deterministic("https://github.com/iterative-test") 204 | 'd5aaeca4-fe6a-5c72-8aa7-6dcd65974973', // deterministic("https://gitlab.com/iterative.ai") 205 | 'b6df227b-5b3d-5190-a8fa-d272b617ee6c', // deterministic("https://gitlab.com/iterative-test") 206 | '2c6415f0-cb5a-5e52-8c81-c5af4f11715d', // deterministic("https://bitbucket.com/iterative-ai") 207 | 'c0b86b90-d63c-5fb0-b84d-718d8e15f8d6' // deterministic("https://bitbucket.com/iterative-test") 208 | ].includes(event.group_id) 209 | ) 210 | return; 211 | 212 | const controller = new AbortController(); 213 | const id = setTimeout(() => controller.abort(), 5 * 1000); 214 | await fetch(endpoint, { 215 | signal: controller.signal, 216 | method: 'POST', 217 | headers: { 218 | 'X-Auth-Token': token, 219 | 'Content-Type': 'application/json' 220 | }, 221 | body: JSON.stringify(event), 222 | agent: new ProxyAgent() 223 | }); 224 | clearInterval(id); 225 | } catch (err) { 226 | logger.debug(`Send analytics failed: ${err.message}`); 227 | } 228 | }; 229 | 230 | exports.deterministic = deterministic; 231 | exports.isCI = isCI; 232 | exports.guessCI = guessCI; 233 | exports.userId = userId; 234 | exports.groupId = groupId; 235 | exports.jitsuEventPayload = jitsuEventPayload; 236 | exports.send = send; 237 | -------------------------------------------------------------------------------- /src/cml.e2e.test.js: -------------------------------------------------------------------------------- 1 | const CML = require('../src/cml').default; 2 | 3 | describe('Github tests', () => { 4 | const OLD_ENV = process.env; 5 | 6 | const { 7 | TEST_GITHUB_TOKEN: TOKEN, 8 | TEST_GITHUB_REPOSITORY: REPO, 9 | TEST_GITHUB_COMMIT: SHA 10 | } = process.env; 11 | 12 | beforeEach(() => { 13 | jest.resetModules(); 14 | 15 | process.env = {}; 16 | process.env.REPO_TOKEN = TOKEN; 17 | }); 18 | 19 | afterAll(() => { 20 | process.env = OLD_ENV; 21 | }); 22 | 23 | test('driver has to be github', async () => { 24 | const cml = new CML({ repo: REPO, token: TOKEN }); 25 | expect(cml.driver).toBe('github'); 26 | }); 27 | 28 | test('Publish image without markdown returns an url', async () => { 29 | const path = `${__dirname}/../assets/logo.png`; 30 | 31 | const output = await new CML().publish({ path }); 32 | 33 | expect(output.startsWith('https://')).toBe(true); 34 | expect(output.includes('cml=png')).toBe(true); 35 | }); 36 | 37 | test('Publish image with markdown', async () => { 38 | const path = `${__dirname}/../assets/logo.png`; 39 | const title = 'my title'; 40 | 41 | const output = await new CML().publish({ path, md: true, title }); 42 | 43 | expect(output.startsWith('![](https://')).toBe(true); 44 | expect(output.endsWith(` "${title}")`)).toBe(true); 45 | expect(output.includes('cml=png')).toBe(true); 46 | }); 47 | 48 | test('Publish image embedded in markdown', async () => { 49 | const path = `${__dirname}/../assets/test.md`; 50 | const title = 'my title'; 51 | 52 | const output = await new CML().publish({ path, md: true, title }); 53 | 54 | expect(output.startsWith(`[${title}](https://`)).toBe(true); 55 | expect(output.endsWith(')')).toBe(true); 56 | expect(output.includes('cml=plain')).toBe(true); 57 | }); 58 | 59 | test('Publish a non image file in markdown', async () => { 60 | const path = `${__dirname}/../assets/logo.pdf`; 61 | const title = 'my title'; 62 | 63 | const output = await new CML().publish({ path, md: true, title }); 64 | 65 | expect(output.startsWith(`[${title}](https://`)).toBe(true); 66 | expect(output.endsWith(')')).toBe(true); 67 | expect(output.includes('cml=pdf')).toBe(true); 68 | }); 69 | 70 | test('Comment should succeed with a valid sha', async () => { 71 | const report = '## Test comment'; 72 | 73 | await new CML({ repo: REPO }).commentCreate({ report, commitSha: SHA }); 74 | }); 75 | 76 | test('Comment should fail with a invalid sha', async () => { 77 | let caughtErr; 78 | try { 79 | const report = '## Test comment'; 80 | const commitSha = 'invalid_sha'; 81 | 82 | await new CML({ repo: REPO }).commentCreate({ report, commitSha }); 83 | } catch (err) { 84 | caughtErr = err.message; 85 | } 86 | 87 | expect(caughtErr).toBe('No commit found for SHA: invalid_sha'); 88 | }); 89 | 90 | test('Runner logs', async () => { 91 | const cml = new CML(); 92 | cml.driver = 'github'; 93 | let logs = await cml.parseRunnerLog({ 94 | data: ` 95 | 2022-06-05 16:25:56Z: Listening for Jobs 96 | 2022-06-05 16:26:35Z: Running job: train 97 | 2022-06-05 16:28:03Z: Job train completed with result: Failed 98 | ` 99 | }); 100 | expect(logs.length).toBe(3); 101 | expect(logs[0].status).toBe('ready'); 102 | expect(logs[1].status).toBe('job_started'); 103 | expect(logs[2].status).toBe('job_ended'); 104 | expect(logs[2].success).toBe(false); 105 | 106 | logs = await cml.parseRunnerLog({ 107 | data: '2022-06-05 16:28:03Z: Job train completed with result: Succeeded' 108 | }); 109 | expect(logs[0].status).toBe('job_ended'); 110 | expect(logs[0].success).toBe(true); 111 | }); 112 | }); 113 | 114 | describe('Gitlab tests', () => { 115 | const OLD_ENV = process.env; 116 | 117 | const { 118 | TEST_GITLAB_TOKEN: TOKEN, 119 | TEST_GITLAB_REPOSITORY: REPO, 120 | TEST_GITLAB_COMMIT: SHA 121 | } = process.env; 122 | 123 | beforeEach(() => { 124 | jest.resetModules(); 125 | 126 | process.env = {}; 127 | process.env.REPO_TOKEN = TOKEN; 128 | }); 129 | 130 | afterAll(() => { 131 | process.env = OLD_ENV; 132 | }); 133 | 134 | test('driver has to be gitlab', async () => { 135 | const cml = new CML({ repo: REPO, token: TOKEN, driver: 'gitlab' }); 136 | expect(cml.driver).toBe('gitlab'); 137 | }); 138 | 139 | test('Publish image using gl without markdown returns an url', async () => { 140 | const path = `${__dirname}/../assets/logo.png`; 141 | 142 | const output = await new CML({ repo: REPO }).publish({ 143 | path, 144 | native: true 145 | }); 146 | 147 | expect(output.startsWith('/uploads/')).toBe(true); 148 | expect(output.includes('cml=png')).toBe(false); 149 | }); 150 | 151 | test('Publish image using gl with markdown', async () => { 152 | const path = `${__dirname}/../assets/logo.png`; 153 | const title = 'my title'; 154 | 155 | const output = await new CML({ repo: REPO }).publish({ 156 | path, 157 | md: true, 158 | title, 159 | native: true 160 | }); 161 | 162 | expect(output.startsWith('![](/uploads/')).toBe(true); 163 | expect(output.endsWith(` "${title}")`)).toBe(true); 164 | expect(output.includes('cml=png')).toBe(false); 165 | }); 166 | 167 | test('Publish a non image file using gl in markdown', async () => { 168 | const path = `${__dirname}/../assets/logo.pdf`; 169 | const title = 'my title'; 170 | 171 | const output = await new CML({ repo: REPO }).publish({ 172 | path, 173 | md: true, 174 | title, 175 | native: true 176 | }); 177 | 178 | expect(output.startsWith(`[${title}](/uploads/`)).toBe(true); 179 | expect(output.endsWith(')')).toBe(true); 180 | expect(output.includes('cml=pdf')).toBe(false); 181 | }); 182 | 183 | test('Publish a non image file using native', async () => { 184 | const path = `${__dirname}/../assets/logo.pdf`; 185 | const title = 'my title'; 186 | 187 | const output = await new CML({ repo: REPO }).publish({ 188 | path, 189 | md: true, 190 | title, 191 | native: true 192 | }); 193 | 194 | expect(output.startsWith(`[${title}](/uploads/`)).toBe(true); 195 | expect(output.endsWith(')')).toBe(true); 196 | expect(output.includes('cml=pdf')).toBe(false); 197 | }); 198 | 199 | test('Publish should fail with an invalid driver', async () => { 200 | let caughtErr; 201 | try { 202 | const path = `${__dirname}/../assets/logo.pdf`; 203 | await new CML({ repo: REPO, driver: 'invalid' }).publish({ 204 | path, 205 | md: true, 206 | native: true 207 | }); 208 | } catch (err) { 209 | caughtErr = err.message; 210 | } 211 | 212 | expect(caughtErr).not.toBeUndefined(); 213 | }); 214 | 215 | test('Comment should succeed with a valid sha', async () => { 216 | const report = '## Test comment'; 217 | await new CML({ repo: REPO }).commentCreate({ report, commitSha: SHA }); 218 | }); 219 | 220 | test('Comment should fail with a invalid sha', async () => { 221 | let caughtErr; 222 | try { 223 | const report = '## Test comment'; 224 | const commitSha = 'invalid_sha'; 225 | 226 | await new CML({ repo: REPO }).commentCreate({ report, commitSha }); 227 | } catch (err) { 228 | caughtErr = err.message; 229 | } 230 | 231 | expect(caughtErr).toBe('Not Found'); 232 | }); 233 | 234 | test('Runner logs', async () => { 235 | const cml = new CML(); 236 | cml.driver = 'gitlab'; 237 | let logs = await cml.parseRunnerLog({ 238 | data: ` 239 | {"level":"info","msg":"Starting runner for https://gitlab.com with token 2SGFrnGt ...","time":"2021-07-02T16:45:05Z"} 240 | {"job":1396213069,"level":"info","msg":"Checking for jobs... received","repo_url":"https://gitlab.com/iterative.ai/fashion_mnist.git","runner":"2SGFrnGt","time":"2021-07-02T16:45:47Z"} 241 | {"duration_s":120.0120526,"job":1396213069,"level":"warning","msg":"Job failed: execution took longer than 2m0s seconds","project":27856642,"runner":"2SGFrnGt","time":"2021-07-02T16:47:47Z"} 242 | ` 243 | }); 244 | expect(logs.length).toBe(3); 245 | expect(logs[0].status).toBe('ready'); 246 | expect(logs[1].status).toBe('job_started'); 247 | expect(logs[2].status).toBe('job_ended'); 248 | expect(logs[2].success).toBe(false); 249 | 250 | logs = await cml.parseRunnerLog({ 251 | data: '{"duration_s":7.706165838,"job":2177867438,"level":"info","msg":"Job succeeded","project":27939020,"runner":"fe36krFK","time":"2022-03-08T18:12:57+01:00"}' 252 | }); 253 | expect(logs[0].status).toBe('job_ended'); 254 | expect(logs[0].success).toBe(true); 255 | }); 256 | }); 257 | 258 | describe('Bitbucket tests', () => { 259 | const OLD_ENV = process.env; 260 | 261 | const { TEST_BITBUCKET_TOKEN: TOKEN = 'DUMMY' } = process.env; 262 | 263 | beforeEach(() => { 264 | jest.resetModules(); 265 | 266 | process.env = {}; 267 | process.env.REPO_TOKEN = TOKEN; 268 | }); 269 | 270 | afterAll(() => { 271 | process.env = OLD_ENV; 272 | }); 273 | 274 | test('Runner logs', async () => { 275 | const cml = new CML(); 276 | cml.driver = 'bitbucket'; 277 | let logs = await cml.parseRunnerLog({ 278 | data: ` 279 | [2022-06-05 17:23:41,945] Updating runner status to "ONLINE" and checking for new steps assigned to the runner after 0 seconds and then every 30 seconds. 280 | [2022-06-05 17:24:12,246] Getting step StepId{accountUuid={XXXXX-XXX-XXX-XXXXXXXX}, repositoryUuid={XXXXX-XXX-XXX-XXXXXXXX}, pipelineUuid={XXXXX-XXX-XXX-XXXXXXXX}, stepUuid={XXXXX-XXX-XXX-XXXXXXXX}}. 281 | [2022-06-05 17:24:53,466] Completing step with result Result{status=FAILED, error=None}. 282 | ` 283 | }); 284 | expect(logs.length).toBe(3); 285 | expect(logs[0].status).toBe('ready'); 286 | expect(logs[1].status).toBe('job_started'); 287 | expect(logs[2].status).toBe('job_ended'); 288 | expect(logs[2].success).toBe(false); 289 | 290 | logs = await cml.parseRunnerLog({ 291 | data: '[2022-06-05 17:24:53,466] Completing step with result Result{status=PASSED, error=None}.' 292 | }); 293 | expect(logs[0].status).toBe('job_ended'); 294 | expect(logs[0].success).toBe(true); 295 | }); 296 | }); 297 | -------------------------------------------------------------------------------- /src/commenttarget.js: -------------------------------------------------------------------------------- 1 | const { logger } = require('./logger'); 2 | 3 | const SEPARATOR = '/'; 4 | 5 | async function parseCommentTarget(opts = {}) { 6 | const { commitSha: commit, pr, target, drv } = opts; 7 | 8 | let commentTarget = target; 9 | // Handle legacy comment target flags. 10 | if (commit) { 11 | drv.warn( 12 | `Deprecation warning: use --target="commit${SEPARATOR}" instead of --commit-sha=` 13 | ); 14 | commentTarget = `commit${SEPARATOR}${commit}`; 15 | } 16 | if (pr) { 17 | drv.warn('Deprecation warning: use --target=pr instead of --pr'); 18 | commentTarget = 'pr'; 19 | } 20 | // Handle comment targets that are incomplete, e.g. 'pr' or 'commit'. 21 | let prNumber; 22 | let commitPr; 23 | switch (commentTarget.toLowerCase()) { 24 | case 'commit': 25 | logger.debug(`Comment target "commit" mapped to "commit/${drv.sha}"`); 26 | return { target: 'commit', commitSha: drv.sha }; 27 | case 'pr': 28 | case 'auto': 29 | // Determine PR id from forge env vars (if we're in a PR context). 30 | prNumber = drv.pr; 31 | if (prNumber) { 32 | logger.debug( 33 | `Comment target "${commentTarget}" mapped to "pr/${prNumber}"` 34 | ); 35 | return { target: 'pr', prNumber: prNumber }; 36 | } 37 | // Or fallback to determining PR by HEAD commit. 38 | // TODO: handle issue with PR HEAD commit not matching source branch in github. 39 | [commitPr = {}] = await drv.commitPrs({ commitSha: drv.sha }); 40 | if (commitPr.url) { 41 | [prNumber] = commitPr.url.split('/').slice(-1); 42 | logger.debug( 43 | `Comment target "${commentTarget}" mapped to "pr/${prNumber}" based on commit "${drv.sha}"` 44 | ); 45 | return { target: 'pr', prNumber }; 46 | } 47 | // If target is 'auto', fallback to issuing commit comments. 48 | if (commentTarget === 'auto') { 49 | logger.debug( 50 | `Comment target "${commentTarget}" mapped to "commit/${drv.sha}"` 51 | ); 52 | return { target: 'commit', commitSha: drv.sha }; 53 | } 54 | throw new Error(`PR for commit sha "${drv.sha}" not found`); 55 | } 56 | // Handle qualified comment targets, e.g. 'issue/id'. 57 | const separatorPos = commentTarget.indexOf(SEPARATOR); 58 | if (separatorPos === -1) { 59 | throw new Error(`Failed to parse comment --target="${commentTarget}"`); 60 | } 61 | const targetType = commentTarget.slice(0, separatorPos); 62 | const id = commentTarget.slice(separatorPos + 1); 63 | switch (targetType.toLowerCase()) { 64 | case 'commit': 65 | return { target: targetType, commitSha: id }; 66 | case 'pr': 67 | return { target: targetType, prNumber: id }; 68 | case 'issue': 69 | return { target: targetType, issueId: id }; 70 | default: 71 | throw new Error(`unsupported comment target "${commentTarget}"`); 72 | } 73 | } 74 | 75 | module.exports = { parseCommentTarget }; 76 | -------------------------------------------------------------------------------- /src/commenttarget.test.js: -------------------------------------------------------------------------------- 1 | const { parseCommentTarget } = require('../src/commenttarget'); 2 | 3 | describe('comment target tests', () => { 4 | beforeEach(() => { 5 | jest.resetModules(); 6 | }); 7 | 8 | test('qualified comment target: pr', async () => { 9 | const target = await parseCommentTarget({ 10 | target: 'pr/3' 11 | }); 12 | expect(target).toEqual({ target: 'pr', prNumber: '3' }); 13 | }); 14 | 15 | test('qualified comment target: commit', async () => { 16 | const target = await parseCommentTarget({ 17 | target: 'commit/abcdefg' 18 | }); 19 | expect(target).toEqual({ target: 'commit', commitSha: 'abcdefg' }); 20 | }); 21 | 22 | test('qualified comment target: issue', async () => { 23 | const target = await parseCommentTarget({ 24 | target: 'issue/3' 25 | }); 26 | expect(target).toEqual({ target: 'issue', issueId: '3' }); 27 | }); 28 | 29 | test('qualified comment target: unsupported', async () => { 30 | try { 31 | await parseCommentTarget({ 32 | target: 'unsupported/3' 33 | }); 34 | } catch (error) { 35 | expect(error.message).toBe('unsupported comment target "unsupported/3"'); 36 | } 37 | }); 38 | 39 | test('legacy flag: commitSha', async () => { 40 | const drv = { warn: () => {} }; 41 | const target = await parseCommentTarget({ 42 | drv, 43 | commitSha: 'abcdefg', 44 | target: 'issue/3' // target will be replaced with commit 45 | }); 46 | expect(target).toEqual({ target: 'commit', commitSha: 'abcdefg' }); 47 | }); 48 | 49 | // Test retrieving the PR id from the driver's context. 50 | test('legacy flag: pr, context', async () => { 51 | const drv = { 52 | warn: () => {}, 53 | pr: '4' // driver returns the PR id from context 54 | }; 55 | const target = await parseCommentTarget({ 56 | drv, 57 | pr: true, 58 | target: 'issue/3' // target will be replaced 59 | }); 60 | expect(target).toEqual({ target: 'pr', prNumber: '4' }); 61 | }); 62 | 63 | // Test calculating the PR id based on the commit sha. 64 | test('legacy flag: pr, commit sha', async () => { 65 | const drv = { 66 | warn: () => {}, 67 | pr: null, // not in PR context 68 | commitPrs: () => [{ url: 'forge/pr/4' }], 69 | sha: 'abcdefg' 70 | }; 71 | const target = await parseCommentTarget({ 72 | drv, 73 | pr: true, 74 | target: 'issue/3' // target will be replaced 75 | }); 76 | expect(target).toEqual({ target: 'pr', prNumber: '4' }); 77 | }); 78 | 79 | // Test using driver supplied commit sha. 80 | test('unqualified flag: commit sha', async () => { 81 | const drv = { 82 | warn: () => {}, 83 | sha: 'abcdefg' 84 | }; 85 | const target = await parseCommentTarget({ 86 | drv, 87 | target: 'commit' 88 | }); 89 | expect(target).toEqual({ target: 'commit', commitSha: 'abcdefg' }); 90 | }); 91 | 92 | // Test retrieving the PR id from the driver's context. 93 | test('unqualified flag: pr, context', async () => { 94 | const drv = { 95 | warn: () => {}, 96 | pr: '4' // driver returns the PR id from context 97 | }; 98 | const target = await parseCommentTarget({ 99 | drv, 100 | target: 'pr' // target will be replaced 101 | }); 102 | expect(target).toEqual({ target: 'pr', prNumber: '4' }); 103 | }); 104 | 105 | // Test calculating the PR id based on the commit sha. 106 | test('unqualified flag: pr, commit sha', async () => { 107 | const drv = { 108 | warn: () => {}, 109 | pr: null, // not in PR context 110 | commitPrs: () => [{ url: 'forge/pr/4' }], 111 | sha: 'abcdefg' 112 | }; 113 | const target = await parseCommentTarget({ 114 | drv, 115 | pr: true, 116 | target: 'pr' // target will be replaced 117 | }); 118 | expect(target).toEqual({ target: 'pr', prNumber: '4' }); 119 | }); 120 | 121 | test('unqualified comment target: issue', async () => { 122 | try { 123 | await parseCommentTarget({ 124 | target: 'issue' 125 | }); 126 | } catch (error) { 127 | expect(error.message).toBe('Failed to parse comment --target="issue"'); 128 | } 129 | }); 130 | 131 | test('auto comment target: pr context', async () => { 132 | const drv = { 133 | warn: () => {}, 134 | pr: '4' // driver returns the PR id from context 135 | }; 136 | 137 | const target = await parseCommentTarget({ 138 | drv, 139 | target: 'auto' 140 | }); 141 | expect(target).toEqual({ target: 'pr', prNumber: '4' }); 142 | }); 143 | 144 | test('auto comment target: pr, commit sha', async () => { 145 | const drv = { 146 | warn: () => {}, 147 | pr: null, // not in PR context 148 | commitPrs: () => [{ url: 'forge/pr/5' }], 149 | sha: 'abcdefg' 150 | }; 151 | 152 | const target = await parseCommentTarget({ 153 | drv, 154 | target: 'auto' 155 | }); 156 | expect(target).toEqual({ target: 'pr', prNumber: '5' }); 157 | }); 158 | 159 | test('auto comment target: fallback commit sha', async () => { 160 | const drv = { 161 | warn: () => {}, 162 | pr: null, // not in PR context 163 | commitPrs: () => [], 164 | sha: 'abcdefg' 165 | }; 166 | 167 | const target = await parseCommentTarget({ 168 | drv, 169 | target: 'auto' 170 | }); 171 | expect(target).toEqual({ target: 'commit', commitSha: 'abcdefg' }); 172 | }); 173 | }); 174 | -------------------------------------------------------------------------------- /src/drivers/bitbucket_cloud.e2e.test.js: -------------------------------------------------------------------------------- 1 | const BitbucketCloud = require('./bitbucket_cloud'); 2 | const { 3 | TEST_BITBUCKET_TOKEN: TOKEN, 4 | TEST_BITBUCKET_REPOSITORY: REPO, 5 | TEST_BITBUCKET_COMMIT: SHA, 6 | TEST_BITBUCKET_ISSUE: ISSUE = 1 7 | } = process.env; 8 | 9 | describe('Non Enviromental tests', () => { 10 | const client = new BitbucketCloud({ repo: REPO, token: TOKEN }); 11 | 12 | test('test repo and token', async () => { 13 | expect(client.repo).toBe(REPO); 14 | expect(client.token).toBe(TOKEN); 15 | }); 16 | 17 | test('Issue comment', async () => { 18 | const report = '## Test comment'; 19 | const issueId = ISSUE; 20 | const url = await client.issueCommentCreate({ report, issueId }); 21 | 22 | expect(url.startsWith('https://')).toBe(true); 23 | }); 24 | 25 | test('Comment', async () => { 26 | const report = '## Test comment'; 27 | const commitSha = SHA; 28 | const url = await client.commitCommentCreate({ report, commitSha }); 29 | 30 | expect(url.startsWith(REPO)).toBe(true); 31 | }); 32 | 33 | test('Check', async () => { 34 | await expect(client.checkCreate()).rejects.toThrow( 35 | 'Bitbucket Cloud does not support check!' 36 | ); 37 | }); 38 | 39 | test('Publish', async () => { 40 | const path = `${__dirname}/../../assets/logo.png`; 41 | const { uri } = await client.upload({ path }); 42 | 43 | expect(uri).not.toBeUndefined(); 44 | }); 45 | 46 | test('Runner token', async () => { 47 | const token = await client.runnerToken(); 48 | await expect(token).toBe('DUMMY'); 49 | }); 50 | 51 | test('updateGitConfig', async () => { 52 | const client = new BitbucketCloud({ 53 | repo: 'http://bitbucket.org/test/test', 54 | token: 'dXNlcjpwYXNz' 55 | }); 56 | const command = await client.updateGitConfig({ 57 | userName: 'john', 58 | userEmail: 'john@test.com', 59 | remote: 'origin' 60 | }); 61 | expect(command).toMatchInlineSnapshot(` 62 | Array [ 63 | Array [ 64 | "git", 65 | "config", 66 | "--unset", 67 | "user.name", 68 | ], 69 | Array [ 70 | "git", 71 | "config", 72 | "--unset", 73 | "user.email", 74 | ], 75 | Array [ 76 | "git", 77 | "config", 78 | "--unset", 79 | "push.default", 80 | ], 81 | Array [ 82 | "git", 83 | "config", 84 | "--unset", 85 | "http.http://bitbucket.org/test/test.proxy", 86 | ], 87 | Array [ 88 | "git", 89 | "config", 90 | "user.name", 91 | "john", 92 | ], 93 | Array [ 94 | "git", 95 | "config", 96 | "user.email", 97 | "john@test.com", 98 | ], 99 | Array [ 100 | "git", 101 | "remote", 102 | "set-url", 103 | "origin", 104 | "https://user:pass@bitbucket.org/test/test", 105 | ], 106 | ] 107 | `); 108 | }); 109 | }); 110 | -------------------------------------------------------------------------------- /src/drivers/bitbucket_cloud.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto'); 2 | const fetch = require('node-fetch'); 3 | const { URL } = require('url'); 4 | const { spawn } = require('child_process'); 5 | const FormData = require('form-data'); 6 | const { ProxyAgent } = require('proxy-agent'); 7 | const { logger } = require('../logger'); 8 | 9 | const { fetchUploadData, exec, gpuPresent, sleep } = require('../utils'); 10 | 11 | const { 12 | BITBUCKET_COMMIT, 13 | BITBUCKET_BRANCH, 14 | BITBUCKET_PIPELINE_UUID, 15 | BITBUCKET_BUILD_NUMBER 16 | } = process.env; 17 | 18 | class BitbucketCloud { 19 | constructor(opts = {}) { 20 | const { repo, token } = opts; 21 | 22 | if (!token) throw new Error('token not found'); 23 | if (!repo) throw new Error('repo not found'); 24 | 25 | this.token = token; 26 | this.repo = repo; 27 | 28 | if (repo !== 'cml') { 29 | const { protocol, host, pathname } = new URL(this.repo); 30 | this.repo_origin = `${protocol}//${host}`; 31 | this.api = 'https://api.bitbucket.org/2.0'; 32 | this.projectPath = encodeURIComponent(pathname.substring(1)); 33 | } 34 | } 35 | 36 | async issueCommentUpsert(opts = {}) { 37 | const { projectPath } = this; 38 | const { issueId, report, id } = opts; 39 | 40 | const endpoint = 41 | `/repositories/${projectPath}/issues/${issueId}/` + 42 | `comments/${id ? id + '/' : ''}`; 43 | return ( 44 | await this.request({ 45 | endpoint, 46 | method: id ? 'PUT' : 'POST', 47 | body: JSON.stringify({ content: { raw: report } }) 48 | }) 49 | ).links.html.href; 50 | } 51 | 52 | async issueCommentCreate(opts = {}) { 53 | const { id, ...rest } = opts; 54 | return this.issueCommentUpsert(rest); 55 | } 56 | 57 | async issueCommentUpdate(opts = {}) { 58 | if (!opts.id) throw new Error('Id is missing updating comment'); 59 | return this.issueCommentUpsert(opts); 60 | } 61 | 62 | async issueComments(opts = {}) { 63 | const { projectPath } = this; 64 | const { issueId } = opts; 65 | 66 | const endpoint = `/repositories/${projectPath}/issues/${issueId}/comments/`; 67 | return (await this.paginatedRequest({ endpoint, method: 'GET' })).map( 68 | ({ id, content: { raw: body = '' } = {} }) => { 69 | return { id, body }; 70 | } 71 | ); 72 | } 73 | 74 | async commitCommentCreate(opts = {}) { 75 | const { projectPath } = this; 76 | const { commitSha, report } = opts; 77 | 78 | const endpoint = `/repositories/${projectPath}/commit/${commitSha}/comments/`; 79 | return ( 80 | await this.request({ 81 | endpoint, 82 | method: 'POST', 83 | body: JSON.stringify({ content: { raw: report } }) 84 | }) 85 | ).links.html.href; 86 | } 87 | 88 | async commitCommentUpdate(opts = {}) { 89 | const { projectPath } = this; 90 | const { commitSha, report, id } = opts; 91 | 92 | const endpoint = `/repositories/${projectPath}/commit/${commitSha}/comments/${id}`; 93 | return ( 94 | await this.request({ 95 | endpoint, 96 | method: 'PUT', 97 | body: JSON.stringify({ content: { raw: report } }) 98 | }) 99 | ).links.html.href; 100 | } 101 | 102 | async commitComments(opts = {}) { 103 | const { projectPath } = this; 104 | const { commitSha } = opts; 105 | 106 | const endpoint = `/repositories/${projectPath}/commit/${commitSha}/comments/`; 107 | 108 | return (await this.paginatedRequest({ endpoint, method: 'GET' })).map( 109 | ({ id, content: { raw: body = '' } = {} }) => { 110 | return { id, body }; 111 | } 112 | ); 113 | } 114 | 115 | async commitPrs(opts = {}) { 116 | const { projectPath } = this; 117 | const { commitSha, state = 'OPEN' } = opts; 118 | 119 | const endpoint = `/repositories/${projectPath}/commit/${commitSha}/pullrequests?state=${state}`; 120 | const prs = await this.paginatedRequest({ endpoint }); 121 | return prs.map((pr) => { 122 | const { 123 | links: { 124 | html: { href: url } 125 | } 126 | } = pr; 127 | return { 128 | url 129 | }; 130 | }); 131 | } 132 | 133 | async checkCreate() { 134 | throw new Error('Bitbucket Cloud does not support check!'); 135 | } 136 | 137 | async upload(opts = {}) { 138 | const { projectPath } = this; 139 | const { size, mime, data } = await fetchUploadData(opts); 140 | 141 | const chunks = []; 142 | for await (const chunk of data) chunks.push(chunk); 143 | const buffer = Buffer.concat(chunks); 144 | 145 | const filename = `cml-${crypto 146 | .createHash('sha256') 147 | .update(buffer) 148 | .digest('hex')}`; 149 | const body = new FormData(); 150 | body.append('files', buffer, { filename }); 151 | 152 | const endpoint = `/repositories/${projectPath}/downloads`; 153 | await this.request({ endpoint, method: 'POST', body }); 154 | return { 155 | uri: `https://bitbucket.org/${decodeURIComponent( 156 | projectPath 157 | )}/downloads/${filename}`, 158 | mime, 159 | size 160 | }; 161 | } 162 | 163 | async runnerToken() { 164 | return 'DUMMY'; 165 | } 166 | 167 | async startRunner(opts) { 168 | const { projectPath } = this; 169 | const { workdir, name, labels, env } = opts; 170 | 171 | logger.warn( 172 | `Bitbucket runner is working under /tmp folder and not under ${workdir} as expected` 173 | ); 174 | 175 | try { 176 | const { uuid: accountId } = await this.request({ endpoint: `/user` }); 177 | const { uuid: repoId } = await this.request({ 178 | endpoint: `/repositories/${projectPath}` 179 | }); 180 | const { 181 | uuid, 182 | oauth_client: { id, secret } 183 | } = await this.registerRunner({ name, labels }); 184 | 185 | const gpu = await gpuPresent(); 186 | const command = `docker container run -t -a stderr -a stdout --rm \ 187 | -v /var/run/docker.sock:/var/run/docker.sock \ 188 | -v /var/lib/docker/containers:/var/lib/docker/containers:ro \ 189 | -v /tmp:/tmp \ 190 | -e ACCOUNT_UUID=${accountId} \ 191 | -e REPOSITORY_UUID=${repoId} \ 192 | -e RUNNER_UUID=${uuid} \ 193 | -e OAUTH_CLIENT_ID=${id} \ 194 | -e OAUTH_CLIENT_SECRET=${secret} \ 195 | -e WORKING_DIRECTORY=/tmp \ 196 | --name ${name} \ 197 | ${gpu ? '--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all' : ''} \ 198 | docker-public.packages.atlassian.com/sox/atlassian/bitbucket-pipelines-runner:1`; 199 | 200 | return spawn(command, { shell: true, env }); 201 | } catch (err) { 202 | throw new Error(`Failed preparing runner: ${err.message}`); 203 | } 204 | } 205 | 206 | async registerRunner(opts = {}) { 207 | const { projectPath } = this; 208 | const { name, labels } = opts; 209 | 210 | const endpoint = `/repositories/${projectPath}/pipelines-config/runners`; 211 | 212 | const request = await this.request({ 213 | api: 'https://api.bitbucket.org/internal', 214 | endpoint, 215 | method: 'POST', 216 | body: JSON.stringify({ 217 | labels: ['self.hosted'].concat(labels.split(',')), 218 | name 219 | }) 220 | }); 221 | 222 | let registered = false; 223 | while (!registered) { 224 | await sleep(1); 225 | const runner = (await this.runners()).find( 226 | (runner) => runner.name === name 227 | ); 228 | if (runner) registered = true; 229 | } 230 | 231 | return request; 232 | } 233 | 234 | async unregisterRunner(opts = {}) { 235 | const { projectPath } = this; 236 | const { runnerId, name } = opts; 237 | const endpoint = `/repositories/${projectPath}/pipelines-config/runners/${runnerId}`; 238 | 239 | try { 240 | await this.request({ 241 | api: 'https://api.bitbucket.org/internal', 242 | endpoint, 243 | method: 'DELETE' 244 | }); 245 | } catch (err) { 246 | if (!err.message.includes('invalid json response body')) { 247 | throw err; 248 | } 249 | } finally { 250 | await exec('docker', 'stop', name); 251 | } 252 | } 253 | 254 | async runners(opts = {}) { 255 | const { projectPath } = this; 256 | 257 | const endpoint = `/repositories/${projectPath}/pipelines-config/runners`; 258 | const runners = await this.paginatedRequest({ 259 | api: 'https://api.bitbucket.org/internal', 260 | endpoint 261 | }); 262 | 263 | return runners.map(({ uuid: id, name, labels, state: { status } }) => ({ 264 | id, 265 | name, 266 | labels, 267 | online: status === 'ONLINE', 268 | busy: status === 'ONLINE' 269 | })); 270 | } 271 | 272 | async runnerById(opts = {}) { 273 | throw new Error('Not yet implemented'); 274 | } 275 | 276 | async prCreate(opts = {}) { 277 | const { projectPath } = this; 278 | const { source, target, title, description, autoMerge } = opts; 279 | 280 | const body = JSON.stringify({ 281 | title, 282 | description, 283 | source: { 284 | branch: { 285 | name: source 286 | } 287 | }, 288 | destination: { 289 | branch: { 290 | name: target 291 | } 292 | } 293 | }); 294 | const endpoint = `/repositories/${projectPath}/pullrequests/`; 295 | const { 296 | id, 297 | links: { 298 | html: { href } 299 | } 300 | } = await this.request({ 301 | method: 'POST', 302 | endpoint, 303 | body 304 | }); 305 | 306 | if (autoMerge) 307 | await this.prAutoMerge({ pullRequestId: id, mergeMode: autoMerge }); 308 | return href; 309 | } 310 | 311 | runnerLogPatterns() { 312 | return { 313 | ready: /Updating runner status to "ONLINE"/, 314 | job_started: /Getting step StepId/, 315 | job_ended: /Completing step with result/, 316 | job_ended_succeded: /Completing step with result Result{status=PASSED/, 317 | pipeline: /pipelineUuid=({.+}), /, 318 | job: /stepUuid=({.+})}/ 319 | }; 320 | } 321 | 322 | async prAutoMerge({ pullRequestId, mergeMode, mergeMessage }) { 323 | logger.warn( 324 | 'Auto-merge is unsupported by Bitbucket Cloud; see https://jira.atlassian.com/browse/BCLOUD-14286. Trying to merge immediately...' 325 | ); 326 | const { projectPath } = this; 327 | const endpoint = `/repositories/${projectPath}/pullrequests/${pullRequestId}/merge`; 328 | const mergeModes = { 329 | merge: 'merge_commit', 330 | rebase: 'fast_forward', 331 | squash: 'squash' 332 | }; 333 | const body = JSON.stringify({ 334 | merge_strategy: mergeModes[mergeMode], 335 | close_source_branch: true, 336 | message: mergeMessage 337 | }); 338 | await this.request({ 339 | method: 'POST', 340 | endpoint, 341 | body 342 | }); 343 | } 344 | 345 | async prCommentCreate(opts = {}) { 346 | const { projectPath } = this; 347 | const { report, prNumber } = opts; 348 | 349 | const endpoint = `/repositories/${projectPath}/pullrequests/${prNumber}/comments/`; 350 | const output = await this.request({ 351 | endpoint, 352 | method: 'POST', 353 | body: JSON.stringify({ content: { raw: report } }) 354 | }); 355 | 356 | return output.links.self.href; 357 | } 358 | 359 | async prCommentUpdate(opts = {}) { 360 | const { projectPath } = this; 361 | const { report, prNumber, id } = opts; 362 | 363 | const endpoint = `/repositories/${projectPath}/pullrequests/${prNumber}/comments/${id}`; 364 | const output = await this.request({ 365 | endpoint, 366 | method: 'PUT', 367 | body: JSON.stringify({ content: { raw: report } }) 368 | }); 369 | 370 | return output.links.self.href; 371 | } 372 | 373 | async prComments(opts = {}) { 374 | const { projectPath } = this; 375 | const { prNumber } = opts; 376 | 377 | const endpoint = `/repositories/${projectPath}/pullrequests/${prNumber}/comments/`; 378 | return (await this.paginatedRequest({ endpoint, method: 'GET' })).map( 379 | ({ id, content: { raw: body = '' } = {} }) => { 380 | return { id, body }; 381 | } 382 | ); 383 | } 384 | 385 | async prs(opts = {}) { 386 | const { projectPath } = this; 387 | const { state = 'OPEN' } = opts; 388 | 389 | try { 390 | const endpoint = `/repositories/${projectPath}/pullrequests?state=${state}`; 391 | const prs = await this.paginatedRequest({ endpoint }); 392 | 393 | return prs.map((pr) => { 394 | const { 395 | links: { 396 | html: { href: url } 397 | }, 398 | source: { 399 | branch: { name: source } 400 | }, 401 | destination: { 402 | branch: { name: target } 403 | } 404 | } = pr; 405 | return { 406 | url, 407 | source, 408 | target 409 | }; 410 | }); 411 | } catch (err) { 412 | if (err.message === 'Not Found Resource not found') 413 | err.message = 414 | "Click 'Go to pull request' on any commit details page to enable this API"; 415 | throw err; 416 | } 417 | } 418 | 419 | async pipelineRerun({ id = BITBUCKET_PIPELINE_UUID, jobId } = {}) { 420 | const { projectPath } = this; 421 | 422 | if (!id && jobId) 423 | logger.warn('BitBucket Cloud does not support pipelineRerun by jobId!'); 424 | 425 | const { target } = await this.request({ 426 | endpoint: `/repositories/${projectPath}/pipelines/${id}`, 427 | method: 'GET' 428 | }); 429 | 430 | await this.request({ 431 | endpoint: `/repositories/${projectPath}/pipelines/`, 432 | method: 'POST', 433 | body: JSON.stringify({ target }) 434 | }); 435 | } 436 | 437 | async pipelineJobs(opts = {}) { 438 | logger.warn('BitBucket Cloud does not support pipelineJobs yet!'); 439 | 440 | return []; 441 | } 442 | 443 | async updateGitConfig({ userName, userEmail, remote } = {}) { 444 | const [user, password] = Buffer.from(this.token, 'base64') 445 | .toString('utf-8') 446 | .split(':'); 447 | 448 | const repo = new URL(this.repo); 449 | repo.password = password; 450 | repo.username = user; 451 | repo.protocol = 'https'; 452 | repo.pathname = repo.pathname.replace('.git', ''); 453 | 454 | const commands = [ 455 | ['git', 'config', '--unset', 'user.name'], 456 | ['git', 'config', '--unset', 'user.email'], 457 | ['git', 'config', '--unset', 'push.default'], 458 | [ 459 | 'git', 460 | 'config', 461 | '--unset', 462 | `http.http://${repo.host}${repo.pathname}.proxy` 463 | ], 464 | ['git', 'config', 'user.name', userName || this.userName], 465 | ['git', 'config', 'user.email', userEmail || this.userEmail], 466 | ['git', 'remote', 'set-url', remote, repo.toString()] 467 | ]; 468 | 469 | return commands; 470 | } 471 | 472 | get workflowId() { 473 | return BITBUCKET_PIPELINE_UUID; 474 | } 475 | 476 | get runId() { 477 | return BITBUCKET_BUILD_NUMBER; 478 | } 479 | 480 | get sha() { 481 | return BITBUCKET_COMMIT; 482 | } 483 | 484 | /** 485 | * Returns the PR number if we're in a PR-related action event. 486 | */ 487 | get pr() { 488 | if ('BITBUCKET_PR_ID' in process.env) { 489 | return process.env.BITBUCKET_PR_ID; 490 | } 491 | return null; 492 | } 493 | 494 | get branch() { 495 | return BITBUCKET_BRANCH; 496 | } 497 | 498 | get userEmail() {} 499 | 500 | get userName() {} 501 | 502 | async paginatedRequest(opts = {}) { 503 | const { method = 'GET', body } = opts; 504 | const { next, values } = await this.request(opts); 505 | 506 | if (next) { 507 | const nextValues = await this.paginatedRequest({ 508 | url: next, 509 | method, 510 | body 511 | }); 512 | values.push(...nextValues); 513 | } 514 | 515 | return values; 516 | } 517 | 518 | async request(opts = {}) { 519 | const { token } = this; 520 | const { url, endpoint, method = 'GET', body, api = this.api } = opts; 521 | 522 | if (!(url || endpoint)) 523 | throw new Error('Bitbucket Cloud API endpoint not found'); 524 | 525 | const headers = { Authorization: `Basic ${token}` }; 526 | if (!body || body.constructor !== FormData) 527 | headers['Content-Type'] = 'application/json'; 528 | 529 | const requestUrl = url || `${api}${endpoint}`; 530 | logger.debug( 531 | `Bitbucket API request, method: ${method}, url: "${requestUrl}"` 532 | ); 533 | const response = await fetch(requestUrl, { 534 | method, 535 | headers, 536 | body, 537 | agent: new ProxyAgent() 538 | }); 539 | 540 | const responseBody = response.headers.get('Content-Type').includes('json') 541 | ? await response.json() 542 | : await response.text(); 543 | 544 | if (!response.ok) { 545 | logger.debug(`Response status is ${response.status}`); 546 | // Attempt to get additional context. We have observed two different error schemas 547 | // from BitBucket API responses: `{"error": {"message": "Error message"}}` and 548 | // `{"error": "Error message"}`, apart from plain text responses like `Bad Request`. 549 | const { error } = responseBody.error 550 | ? responseBody 551 | : { error: responseBody }; 552 | throw new Error( 553 | `${response.statusText} ${error.message || error}`.trim() 554 | ); 555 | } 556 | 557 | return responseBody; 558 | } 559 | 560 | warn(message) { 561 | logger.warn(message); 562 | } 563 | } 564 | 565 | module.exports = BitbucketCloud; 566 | -------------------------------------------------------------------------------- /src/drivers/github.e2e.test.js: -------------------------------------------------------------------------------- 1 | const GithubClient = require('./github'); 2 | 3 | const { 4 | TEST_GITHUB_TOKEN: TOKEN, 5 | TEST_GITHUB_REPOSITORY: REPO, 6 | TEST_GITHUB_COMMIT: SHA, 7 | TEST_GITHUB_ISSUE: ISSUE = 1 8 | } = process.env; 9 | 10 | describe('Non Enviromental tests', () => { 11 | const client = new GithubClient({ repo: REPO, token: TOKEN }); 12 | 13 | test('test repo and token', async () => { 14 | expect(client.repo).toBe(REPO); 15 | expect(client.token).toBe(TOKEN); 16 | 17 | const { owner, repo } = client.ownerRepo(); 18 | const parts = REPO.split('/'); 19 | expect(owner).toBe(parts[parts.length - 2]); 20 | expect(repo).toBe(parts[parts.length - 1]); 21 | }); 22 | 23 | test('Issue comment', async () => { 24 | const report = '## Test comment'; 25 | const issueId = ISSUE; 26 | const url = await client.issueCommentCreate({ issueId, report }); 27 | 28 | expect(url.startsWith(REPO)).toBe(true); 29 | }); 30 | 31 | test('Comment', async () => { 32 | const report = '## Test comment'; 33 | const commitSha = SHA; 34 | const url = await client.commitCommentCreate({ report, commitSha }); 35 | 36 | expect(url.startsWith(REPO)).toBe(true); 37 | }); 38 | 39 | test('Publish', async () => { 40 | await expect(client.upload()).rejects.toThrow( 41 | 'Github does not support publish!' 42 | ); 43 | }); 44 | 45 | test('Runner token', async () => { 46 | const output = await client.runnerToken(); 47 | expect(output.length).toBe(29); 48 | }); 49 | 50 | test('updateGitConfig', async () => { 51 | const client = new GithubClient({ 52 | repo: 'https://github.com/test/test', 53 | token: 'dXNlcjpwYXNz' 54 | }); 55 | const command = await client.updateGitConfig({ remote: 'origin' }); 56 | expect(command).toMatchInlineSnapshot(` 57 | Array [ 58 | Array [ 59 | "git", 60 | "config", 61 | "--unset", 62 | "http.https://github.com/.extraheader", 63 | ], 64 | Array [ 65 | "git", 66 | "config", 67 | "user.name", 68 | "GitHub Action", 69 | ], 70 | Array [ 71 | "git", 72 | "config", 73 | "user.email", 74 | "action@github.com", 75 | ], 76 | Array [ 77 | "git", 78 | "remote", 79 | "set-url", 80 | "origin", 81 | "https://token:dXNlcjpwYXNz@github.com/test/test.git", 82 | ], 83 | ] 84 | `); 85 | }); 86 | 87 | test('Check pinned version of Octokit', async () => { 88 | // This test is a must to ensure that @actions/github is not updated. 89 | // There is a bug that after a reRunWorkflow deprecation rest the library does not contains 90 | // nor the original reRunWorkflow nor the new one! 91 | 92 | const { dependencies } = require('../../package.json'); 93 | expect(dependencies['@actions/github']).toBe('^4.0.0'); 94 | expect(dependencies['@octokit/rest']).toBe('18.0.0'); 95 | expect(dependencies['@octokit/core']).toBe('^3.5.1'); 96 | }); 97 | }); 98 | -------------------------------------------------------------------------------- /src/drivers/gitlab.e2e.test.js: -------------------------------------------------------------------------------- 1 | const GitlabClient = require('./gitlab'); 2 | 3 | const { 4 | TEST_GITLAB_TOKEN: TOKEN, 5 | TEST_GITLAB_REPOSITORY: REPO, 6 | TEST_GITLAB_COMMIT: SHA, 7 | TEST_GITLAB_ISSUE: ISSUE = 1 8 | } = process.env; 9 | 10 | describe('Non Enviromental tests', () => { 11 | const client = new GitlabClient({ repo: REPO, token: TOKEN }); 12 | 13 | test('test repo and token', async () => { 14 | expect(client.repo).toBe(REPO); 15 | expect(client.token).toBe(TOKEN); 16 | }); 17 | 18 | test('Issue comment', async () => { 19 | const report = '## Test comment'; 20 | const issueId = ISSUE; 21 | const url = await client.issueCommentCreate({ issueId, report }); 22 | 23 | expect(url.startsWith(REPO)).toBe(true); 24 | }); 25 | 26 | test('Comment', async () => { 27 | const report = '## Test comment'; 28 | const commitSha = SHA; 29 | const url = await client.commitCommentCreate({ 30 | report, 31 | commitSha 32 | }); 33 | 34 | expect(url.startsWith('https://')).toBe(true); 35 | }); 36 | 37 | test('Check', async () => { 38 | await expect(client.checkCreate()).rejects.toThrow( 39 | 'Gitlab does not support check!' 40 | ); 41 | }); 42 | 43 | test('Publish', async () => { 44 | const path = `${__dirname}/../../assets/logo.png`; 45 | const { uri } = await client.upload({ path }); 46 | 47 | expect(uri).not.toBeUndefined(); 48 | }); 49 | 50 | test('Runner token', async () => { 51 | const output = await client.runnerToken(); 52 | expect(output.length >= 20).toBe(true); 53 | }); 54 | 55 | test('updateGitConfig', async () => { 56 | const client = new GitlabClient({ 57 | repo: 'https://gitlab.com/test/test', 58 | token: 'dXNlcjpwYXNz' 59 | }); 60 | const command = await client.updateGitConfig({ 61 | userName: 'john', 62 | userEmail: 'john@test.com', 63 | remote: 'origin' 64 | }); 65 | expect(command).toMatchInlineSnapshot(` 66 | Array [ 67 | Array [ 68 | "git", 69 | "config", 70 | "user.name", 71 | "john", 72 | ], 73 | Array [ 74 | "git", 75 | "config", 76 | "user.email", 77 | "john@test.com", 78 | ], 79 | Array [ 80 | "git", 81 | "remote", 82 | "set-url", 83 | "origin", 84 | "https://token:dXNlcjpwYXNz@gitlab.com/test/test.git", 85 | ], 86 | ] 87 | `); 88 | }); 89 | }); 90 | -------------------------------------------------------------------------------- /src/logger.js: -------------------------------------------------------------------------------- 1 | const logger = require('winston'); 2 | 3 | const setupLogger = (opts) => { 4 | const { log: level, silent } = opts; 5 | 6 | logger.configure({ 7 | format: process.stdout.isTTY 8 | ? logger.format.combine( 9 | logger.format.colorize({ all: true }), 10 | logger.format.simple() 11 | ) 12 | : logger.format.combine( 13 | logger.format.errors({ stack: true }), 14 | logger.format.json() 15 | ), 16 | transports: [ 17 | new logger.transports.Console({ 18 | stderrLevels: Object.keys(logger.config.npm.levels), 19 | handleExceptions: true, 20 | handleRejections: true, 21 | level, 22 | silent 23 | }) 24 | ] 25 | }); 26 | }; 27 | 28 | if (typeof jest !== 'undefined') { 29 | setupLogger({ log: 'debug', silent: true }); 30 | } 31 | 32 | module.exports = { logger, setupLogger }; 33 | -------------------------------------------------------------------------------- /src/terraform.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs').promises; 2 | const { ltr } = require('semver'); 3 | const { logger } = require('./logger'); 4 | const { exec, tfCapture } = require('./utils'); 5 | 6 | const MIN_TF_VER = '0.14.0'; 7 | 8 | const version = async () => { 9 | try { 10 | const output = await exec('terraform', 'version', '-json'); 11 | const { terraform_version: ver } = JSON.parse(output); 12 | return ver; 13 | } catch (err) { 14 | const output = await exec('terraform', 'version'); 15 | const matches = output.match(/Terraform v(\d{1,2}\.\d{1,2}\.\d{1,2})/); 16 | 17 | if (matches.length < 2) throw new Error('Unable to get TF version'); 18 | 19 | return matches[1]; 20 | } 21 | }; 22 | 23 | const loadTfState = async (opts = {}) => { 24 | const { path } = opts; 25 | const json = await fs.readFile(path, 'utf-8'); 26 | return JSON.parse(json); 27 | }; 28 | 29 | const saveTfState = async (opts = {}) => { 30 | const { path, tfstate } = opts; 31 | await fs.writeFile(path, JSON.stringify(tfstate, null, '\t')); 32 | }; 33 | 34 | const init = async (opts = {}) => { 35 | const { dir = './' } = opts; 36 | return await exec('terraform', `-chdir=${dir}`, 'init'); 37 | }; 38 | 39 | const apply = async (opts = {}) => { 40 | const { dir = './' } = opts; 41 | const { env } = process; 42 | if (env.TF_LOG_PROVIDER === undefined) env.TF_LOG_PROVIDER = 'DEBUG'; 43 | try { 44 | await tfCapture( 45 | 'terraform', 46 | [`-chdir=${dir}`, 'apply', '-auto-approve', '-json'], 47 | { 48 | cwd: process.cwd(), 49 | env, 50 | shell: true 51 | } 52 | ); 53 | } catch (rejectionLogs) { 54 | process.stdout.write(rejectionLogs); 55 | throw new Error('terraform apply error'); 56 | } 57 | }; 58 | 59 | const destroy = async (opts = {}) => { 60 | const { dir = './', target } = opts; 61 | return await exec( 62 | 'terraform', 63 | `-chdir=${dir}`, 64 | 'destroy', 65 | '-auto-approve', 66 | ...(target ? ['-target', target] : []) 67 | ); 68 | }; 69 | 70 | const iterativeProviderTpl = ({ tpiVersion }) => ({ 71 | terraform: { 72 | required_providers: { 73 | iterative: { 74 | source: 'iterative/iterative', 75 | ...(tpiVersion && { version: tpiVersion }) 76 | } 77 | } 78 | }, 79 | provider: { 80 | iterative: {} 81 | } 82 | }); 83 | 84 | const iterativeCmlRunnerTpl = (opts = {}) => { 85 | const tfObj = { 86 | ...iterativeProviderTpl(opts), 87 | resource: { 88 | iterative_cml_runner: { 89 | runner: { 90 | ...(opts.image && { image: opts.image }), 91 | ...(opts.awsSecurityGroup && { 92 | aws_security_group: opts.awsSecurityGroup 93 | }), 94 | ...(opts.awsSubnet && { aws_subnet_id: opts.awsSubnet }), 95 | ...(opts.cloud && { cloud: opts.cloud }), 96 | ...(opts.cmlVersion && { cml_version: opts.cmlVersion }), 97 | ...(opts.dockerVolumes && { docker_volumes: opts.dockerVolumes }), 98 | ...(opts.driver && { driver: opts.driver }), 99 | ...(opts.gpu && { instance_gpu: opts.gpu }), 100 | ...(opts.hddSize && { instance_hdd_size: opts.hddSize }), 101 | ...(typeof opts.idleTimeout !== 'undefined' && { 102 | idle_timeout: opts.idleTimeout 103 | }), 104 | ...(opts.labels && { labels: opts.labels }), 105 | ...(opts.metadata && { metadata: opts.metadata }), 106 | ...(opts.name && { name: opts.name }), 107 | ...(opts.permissionSet && { 108 | instance_permission_set: opts.permissionSet 109 | }), 110 | ...(opts.region && { region: opts.region }), 111 | ...(opts.repo && { repo: opts.repo }), 112 | ...(opts.single && { single: opts.single }), 113 | ...(opts.spot && { spot: opts.spot }), 114 | ...(opts.spotPrice && { spot_price: opts.spotPrice }), 115 | ...(opts.sshPrivate && { ssh_private: opts.sshPrivate }), 116 | ...(opts.startupScript && { startup_script: opts.startupScript }), 117 | ...(opts.token && { token: opts.token }), 118 | ...(opts.type && { instance_type: opts.type }), 119 | ...(opts.kubernetesNodeSelector && { 120 | kubernetes_node_selector: opts.kubernetesNodeSelector 121 | }) 122 | } 123 | } 124 | } 125 | }; 126 | logger.debug(`terraform data: ${JSON.stringify(tfObj)}`); 127 | return tfObj; 128 | }; 129 | 130 | const checkMinVersion = async () => { 131 | const ver = await version(); 132 | if (ltr(ver, MIN_TF_VER)) 133 | throw new Error( 134 | `Terraform version must be greater that 14: current ${ver}` 135 | ); 136 | }; 137 | 138 | exports.version = version; 139 | exports.loadTfState = loadTfState; 140 | exports.saveTfState = saveTfState; 141 | exports.init = init; 142 | exports.apply = apply; 143 | exports.destroy = destroy; 144 | exports.iterativeProviderTpl = iterativeProviderTpl; 145 | exports.iterativeCmlRunnerTpl = iterativeCmlRunnerTpl; 146 | exports.checkMinVersion = checkMinVersion; 147 | -------------------------------------------------------------------------------- /src/terraform.test.js: -------------------------------------------------------------------------------- 1 | const { iterativeCmlRunnerTpl } = require('./terraform'); 2 | 3 | describe('Terraform tests', () => { 4 | test('default options', async () => { 5 | const output = iterativeCmlRunnerTpl({}); 6 | expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(` 7 | "{ 8 | \\"terraform\\": { 9 | \\"required_providers\\": { 10 | \\"iterative\\": { 11 | \\"source\\": \\"iterative/iterative\\" 12 | } 13 | } 14 | }, 15 | \\"provider\\": { 16 | \\"iterative\\": {} 17 | }, 18 | \\"resource\\": { 19 | \\"iterative_cml_runner\\": { 20 | \\"runner\\": {} 21 | } 22 | } 23 | }" 24 | `); 25 | }); 26 | 27 | test('basic settings', async () => { 28 | const output = iterativeCmlRunnerTpl({ 29 | repo: 'https://', 30 | token: 'abc', 31 | driver: 'gitlab', 32 | labels: 'mylabel', 33 | idleTimeout: 300, 34 | name: 'myrunner', 35 | single: true, 36 | cloud: 'aws', 37 | region: 'west', 38 | type: 'mymachinetype', 39 | gpu: 'mygputype', 40 | hddSize: 50, 41 | sshPrivate: 'myprivate', 42 | spot: true, 43 | spotPrice: '0.0001', 44 | awsSecurityGroup: 'mysg' 45 | }); 46 | expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(` 47 | "{ 48 | \\"terraform\\": { 49 | \\"required_providers\\": { 50 | \\"iterative\\": { 51 | \\"source\\": \\"iterative/iterative\\" 52 | } 53 | } 54 | }, 55 | \\"provider\\": { 56 | \\"iterative\\": {} 57 | }, 58 | \\"resource\\": { 59 | \\"iterative_cml_runner\\": { 60 | \\"runner\\": { 61 | \\"aws_security_group\\": \\"mysg\\", 62 | \\"cloud\\": \\"aws\\", 63 | \\"driver\\": \\"gitlab\\", 64 | \\"instance_gpu\\": \\"mygputype\\", 65 | \\"instance_hdd_size\\": 50, 66 | \\"idle_timeout\\": 300, 67 | \\"labels\\": \\"mylabel\\", 68 | \\"name\\": \\"myrunner\\", 69 | \\"region\\": \\"west\\", 70 | \\"repo\\": \\"https://\\", 71 | \\"single\\": true, 72 | \\"spot\\": true, 73 | \\"spot_price\\": \\"0.0001\\", 74 | \\"ssh_private\\": \\"myprivate\\", 75 | \\"token\\": \\"abc\\", 76 | \\"instance_type\\": \\"mymachinetype\\" 77 | } 78 | } 79 | } 80 | }" 81 | `); 82 | }); 83 | 84 | test('basic settings with runner forever', async () => { 85 | const output = iterativeCmlRunnerTpl({ 86 | repo: 'https://', 87 | token: 'abc', 88 | driver: 'gitlab', 89 | labels: 'mylabel', 90 | idleTimeout: 0, 91 | name: 'myrunner', 92 | single: true, 93 | cloud: 'aws', 94 | region: 'west', 95 | type: 'mymachinetype', 96 | gpu: 'mygputype', 97 | hddSize: 50, 98 | sshPrivate: 'myprivate', 99 | spot: true, 100 | spotPrice: '0.0001' 101 | }); 102 | expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(` 103 | "{ 104 | \\"terraform\\": { 105 | \\"required_providers\\": { 106 | \\"iterative\\": { 107 | \\"source\\": \\"iterative/iterative\\" 108 | } 109 | } 110 | }, 111 | \\"provider\\": { 112 | \\"iterative\\": {} 113 | }, 114 | \\"resource\\": { 115 | \\"iterative_cml_runner\\": { 116 | \\"runner\\": { 117 | \\"cloud\\": \\"aws\\", 118 | \\"driver\\": \\"gitlab\\", 119 | \\"instance_gpu\\": \\"mygputype\\", 120 | \\"instance_hdd_size\\": 50, 121 | \\"idle_timeout\\": 0, 122 | \\"labels\\": \\"mylabel\\", 123 | \\"name\\": \\"myrunner\\", 124 | \\"region\\": \\"west\\", 125 | \\"repo\\": \\"https://\\", 126 | \\"single\\": true, 127 | \\"spot\\": true, 128 | \\"spot_price\\": \\"0.0001\\", 129 | \\"ssh_private\\": \\"myprivate\\", 130 | \\"token\\": \\"abc\\", 131 | \\"instance_type\\": \\"mymachinetype\\" 132 | } 133 | } 134 | } 135 | }" 136 | `); 137 | }); 138 | 139 | test('basic settings with metadata', async () => { 140 | const output = iterativeCmlRunnerTpl({ 141 | repo: 'https://', 142 | token: 'abc', 143 | driver: 'gitlab', 144 | labels: 'mylabel', 145 | idleTimeout: 300, 146 | name: 'myrunner', 147 | single: true, 148 | cloud: 'aws', 149 | region: 'west', 150 | type: 'mymachinetype', 151 | gpu: 'mygputype', 152 | hddSize: 50, 153 | sshPrivate: 'myprivate', 154 | spot: true, 155 | spotPrice: '0.0001', 156 | metadata: { one: 'value', two: null, 'no problem': 'with spaces' } 157 | }); 158 | expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(` 159 | "{ 160 | \\"terraform\\": { 161 | \\"required_providers\\": { 162 | \\"iterative\\": { 163 | \\"source\\": \\"iterative/iterative\\" 164 | } 165 | } 166 | }, 167 | \\"provider\\": { 168 | \\"iterative\\": {} 169 | }, 170 | \\"resource\\": { 171 | \\"iterative_cml_runner\\": { 172 | \\"runner\\": { 173 | \\"cloud\\": \\"aws\\", 174 | \\"driver\\": \\"gitlab\\", 175 | \\"instance_gpu\\": \\"mygputype\\", 176 | \\"instance_hdd_size\\": 50, 177 | \\"idle_timeout\\": 300, 178 | \\"labels\\": \\"mylabel\\", 179 | \\"metadata\\": { 180 | \\"one\\": \\"value\\", 181 | \\"two\\": null, 182 | \\"no problem\\": \\"with spaces\\" 183 | }, 184 | \\"name\\": \\"myrunner\\", 185 | \\"region\\": \\"west\\", 186 | \\"repo\\": \\"https://\\", 187 | \\"single\\": true, 188 | \\"spot\\": true, 189 | \\"spot_price\\": \\"0.0001\\", 190 | \\"ssh_private\\": \\"myprivate\\", 191 | \\"token\\": \\"abc\\", 192 | \\"instance_type\\": \\"mymachinetype\\" 193 | } 194 | } 195 | } 196 | }" 197 | `); 198 | }); 199 | 200 | test('basic settings with kubernetes node selector', async () => { 201 | const output = iterativeCmlRunnerTpl({ 202 | repo: 'https://', 203 | token: 'abc', 204 | driver: 'gitlab', 205 | labels: 'mylabel', 206 | idleTimeout: 300, 207 | name: 'myrunner', 208 | single: true, 209 | cloud: 'aws', 210 | region: 'west', 211 | type: 'mymachinetype', 212 | gpu: 'mygputype', 213 | hddSize: 50, 214 | sshPrivate: 'myprivate', 215 | spot: true, 216 | spotPrice: '0.0001', 217 | kubernetesNodeSelector: { 218 | accelerator: 'infer', 219 | ram: null, 220 | 'disk type': 'hard drives' 221 | } 222 | }); 223 | expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(` 224 | "{ 225 | \\"terraform\\": { 226 | \\"required_providers\\": { 227 | \\"iterative\\": { 228 | \\"source\\": \\"iterative/iterative\\" 229 | } 230 | } 231 | }, 232 | \\"provider\\": { 233 | \\"iterative\\": {} 234 | }, 235 | \\"resource\\": { 236 | \\"iterative_cml_runner\\": { 237 | \\"runner\\": { 238 | \\"cloud\\": \\"aws\\", 239 | \\"driver\\": \\"gitlab\\", 240 | \\"instance_gpu\\": \\"mygputype\\", 241 | \\"instance_hdd_size\\": 50, 242 | \\"idle_timeout\\": 300, 243 | \\"labels\\": \\"mylabel\\", 244 | \\"name\\": \\"myrunner\\", 245 | \\"region\\": \\"west\\", 246 | \\"repo\\": \\"https://\\", 247 | \\"single\\": true, 248 | \\"spot\\": true, 249 | \\"spot_price\\": \\"0.0001\\", 250 | \\"ssh_private\\": \\"myprivate\\", 251 | \\"token\\": \\"abc\\", 252 | \\"instance_type\\": \\"mymachinetype\\", 253 | \\"kubernetes_node_selector\\": { 254 | \\"accelerator\\": \\"infer\\", 255 | \\"ram\\": null, 256 | \\"disk type\\": \\"hard drives\\" 257 | } 258 | } 259 | } 260 | } 261 | }" 262 | `); 263 | }); 264 | 265 | test('basic settings with docker volumes', async () => { 266 | const output = iterativeCmlRunnerTpl({ 267 | repo: 'https://', 268 | token: 'abc', 269 | driver: 'gitlab', 270 | labels: 'mylabel', 271 | idleTimeout: 300, 272 | name: 'myrunner', 273 | single: true, 274 | cloud: 'aws', 275 | region: 'west', 276 | type: 'mymachinetype', 277 | gpu: 'mygputype', 278 | hddSize: 50, 279 | sshPrivate: 'myprivate', 280 | spot: true, 281 | spotPrice: '0.0001', 282 | dockerVolumes: ['/aa:/aa', '/bb:/bb'] 283 | }); 284 | expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(` 285 | "{ 286 | \\"terraform\\": { 287 | \\"required_providers\\": { 288 | \\"iterative\\": { 289 | \\"source\\": \\"iterative/iterative\\" 290 | } 291 | } 292 | }, 293 | \\"provider\\": { 294 | \\"iterative\\": {} 295 | }, 296 | \\"resource\\": { 297 | \\"iterative_cml_runner\\": { 298 | \\"runner\\": { 299 | \\"cloud\\": \\"aws\\", 300 | \\"docker_volumes\\": [ 301 | \\"/aa:/aa\\", 302 | \\"/bb:/bb\\" 303 | ], 304 | \\"driver\\": \\"gitlab\\", 305 | \\"instance_gpu\\": \\"mygputype\\", 306 | \\"instance_hdd_size\\": 50, 307 | \\"idle_timeout\\": 300, 308 | \\"labels\\": \\"mylabel\\", 309 | \\"name\\": \\"myrunner\\", 310 | \\"region\\": \\"west\\", 311 | \\"repo\\": \\"https://\\", 312 | \\"single\\": true, 313 | \\"spot\\": true, 314 | \\"spot_price\\": \\"0.0001\\", 315 | \\"ssh_private\\": \\"myprivate\\", 316 | \\"token\\": \\"abc\\", 317 | \\"instance_type\\": \\"mymachinetype\\" 318 | } 319 | } 320 | } 321 | }" 322 | `); 323 | }); 324 | 325 | test('basic settings with permission set', async () => { 326 | const output = iterativeCmlRunnerTpl({ 327 | repo: 'https://', 328 | token: 'abc', 329 | driver: 'gitlab', 330 | labels: 'mylabel', 331 | idleTimeout: 300, 332 | name: 'myrunner', 333 | single: true, 334 | cloud: 'aws', 335 | region: 'west', 336 | type: 'mymachinetype', 337 | permissionSet: 'arn:aws:iam::1:instance-profile/x', 338 | gpu: 'mygputype', 339 | hddSize: 50, 340 | sshPrivate: 'myprivate', 341 | spot: true, 342 | spotPrice: '0.0001', 343 | awsSecurityGroup: 'mysg' 344 | }); 345 | expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(` 346 | "{ 347 | \\"terraform\\": { 348 | \\"required_providers\\": { 349 | \\"iterative\\": { 350 | \\"source\\": \\"iterative/iterative\\" 351 | } 352 | } 353 | }, 354 | \\"provider\\": { 355 | \\"iterative\\": {} 356 | }, 357 | \\"resource\\": { 358 | \\"iterative_cml_runner\\": { 359 | \\"runner\\": { 360 | \\"aws_security_group\\": \\"mysg\\", 361 | \\"cloud\\": \\"aws\\", 362 | \\"driver\\": \\"gitlab\\", 363 | \\"instance_gpu\\": \\"mygputype\\", 364 | \\"instance_hdd_size\\": 50, 365 | \\"idle_timeout\\": 300, 366 | \\"labels\\": \\"mylabel\\", 367 | \\"name\\": \\"myrunner\\", 368 | \\"instance_permission_set\\": \\"arn:aws:iam::1:instance-profile/x\\", 369 | \\"region\\": \\"west\\", 370 | \\"repo\\": \\"https://\\", 371 | \\"single\\": true, 372 | \\"spot\\": true, 373 | \\"spot_price\\": \\"0.0001\\", 374 | \\"ssh_private\\": \\"myprivate\\", 375 | \\"token\\": \\"abc\\", 376 | \\"instance_type\\": \\"mymachinetype\\" 377 | } 378 | } 379 | } 380 | }" 381 | `); 382 | }); 383 | 384 | test('Startup script', async () => { 385 | const output = iterativeCmlRunnerTpl({ 386 | repo: 'https://', 387 | token: 'abc', 388 | driver: 'gitlab', 389 | labels: 'mylabel', 390 | idleTimeout: 300, 391 | name: 'myrunner', 392 | single: true, 393 | cloud: 'aws', 394 | region: 'west', 395 | type: 'mymachinetype', 396 | gpu: 'mygputype', 397 | hddSize: 50, 398 | sshPrivate: 'myprivate', 399 | spot: true, 400 | spotPrice: '0.0001', 401 | startupScript: 'c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=' 402 | }); 403 | expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(` 404 | "{ 405 | \\"terraform\\": { 406 | \\"required_providers\\": { 407 | \\"iterative\\": { 408 | \\"source\\": \\"iterative/iterative\\" 409 | } 410 | } 411 | }, 412 | \\"provider\\": { 413 | \\"iterative\\": {} 414 | }, 415 | \\"resource\\": { 416 | \\"iterative_cml_runner\\": { 417 | \\"runner\\": { 418 | \\"cloud\\": \\"aws\\", 419 | \\"driver\\": \\"gitlab\\", 420 | \\"instance_gpu\\": \\"mygputype\\", 421 | \\"instance_hdd_size\\": 50, 422 | \\"idle_timeout\\": 300, 423 | \\"labels\\": \\"mylabel\\", 424 | \\"name\\": \\"myrunner\\", 425 | \\"region\\": \\"west\\", 426 | \\"repo\\": \\"https://\\", 427 | \\"single\\": true, 428 | \\"spot\\": true, 429 | \\"spot_price\\": \\"0.0001\\", 430 | \\"ssh_private\\": \\"myprivate\\", 431 | \\"startup_script\\": \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\", 432 | \\"token\\": \\"abc\\", 433 | \\"instance_type\\": \\"mymachinetype\\" 434 | } 435 | } 436 | } 437 | }" 438 | `); 439 | }); 440 | }); 441 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const PATH = require('path'); 3 | const { Buffer } = require('buffer'); 4 | const fetch = require('node-fetch'); 5 | const { ProxyAgent } = require('proxy-agent'); 6 | const NodeSSH = require('node-ssh').NodeSSH; 7 | const stripAnsi = require('strip-ansi'); 8 | const { logger } = require('./logger'); 9 | const uuid = require('uuid'); 10 | const getOS = require('getos'); 11 | 12 | const { FileMagic, MagicFlags } = require('@npcz/magic'); 13 | const tempy = require('tempy'); 14 | 15 | const getos = async () => { 16 | return new Promise((resolve, reject) => { 17 | getOS((err, os) => { 18 | if (err) reject(err); 19 | resolve(os); 20 | }); 21 | }); 22 | }; 23 | 24 | const waitForever = () => new Promise((resolve) => resolve); 25 | 26 | const exec = async (file, ...args) => { 27 | return new Promise((resolve, reject) => { 28 | require('child_process').execFile( 29 | file, 30 | args, 31 | { ...process.env }, 32 | (error, stdout, stderr) => { 33 | if (!process.stdout.isTTY) { 34 | stdout = stripAnsi(stdout); 35 | stderr = stripAnsi(stderr); 36 | } 37 | if (error) 38 | reject(new Error(`${[file, ...args]}\n\t${stdout}\n\t${stderr}`)); 39 | 40 | resolve((stdout || stderr).slice(0, -1)); 41 | } 42 | ); 43 | }); 44 | }; 45 | 46 | const mimeType = async (opts) => { 47 | const { path, buffer } = opts; 48 | const magicFile = PATH.join(__dirname, '../assets/magic.mgc'); 49 | if (fs.existsSync(magicFile)) FileMagic.magicFile = magicFile; 50 | const fileMagic = await FileMagic.getInstance(); 51 | 52 | let tmppath; 53 | if (buffer) { 54 | tmppath = tempy.file(); 55 | await fs.promises.writeFile(tmppath, buffer); 56 | } 57 | 58 | const [mime] = ( 59 | fileMagic.detect( 60 | buffer ? tmppath : path, 61 | fileMagic.flags | MagicFlags.MAGIC_MIME 62 | ) || [] 63 | ).split(';'); 64 | 65 | return mime; 66 | }; 67 | 68 | const fetchUploadData = async (opts) => { 69 | const { path, buffer, mimeType: mimeTypeIn } = opts; 70 | 71 | const size = path ? (await fs.promises.stat(path)).size : buffer.length; 72 | const data = path ? fs.createReadStream(path) : buffer; 73 | const mime = mimeTypeIn || (await mimeType(opts)); 74 | return { mime, size, data }; 75 | }; 76 | 77 | const upload = async (opts) => { 78 | const { path, session, url = 'https://asset.cml.dev' } = opts; 79 | 80 | const { mime, size, data: body } = await fetchUploadData(opts); 81 | const filename = path ? PATH.basename(path) : `file.${mime.split('/')[1]}`; 82 | 83 | const headers = { 84 | 'Content-Length': size, 85 | 'Content-Type': mime, 86 | 'Content-Disposition': `inline; filename="${filename}"` 87 | }; 88 | 89 | if (session) headers['Content-Address-Seed'] = `${session}:${path}`; 90 | const response = await fetch(url, { method: 'POST', headers, body }); 91 | const uri = await response.text(); 92 | 93 | if (!uri) 94 | throw new Error( 95 | `Empty response from asset backend with status code ${response.status}` 96 | ); 97 | 98 | return { uri, mime, size }; 99 | }; 100 | 101 | const randid = () => { 102 | return ( 103 | Math.random().toString(36).substring(2, 7) + 104 | Math.random().toString(36).substring(2, 7) 105 | ); 106 | }; 107 | 108 | const sleep = (secs) => { 109 | return new Promise((resolve) => { 110 | setTimeout(resolve, secs * 1000); 111 | }); 112 | }; 113 | 114 | const isProcRunning = async (opts) => { 115 | const { name } = opts; 116 | 117 | const cmd = (() => { 118 | switch (process.platform) { 119 | case 'win32': 120 | return `tasklist`; 121 | case 'darwin': 122 | return `ps -ax`; 123 | case 'linux': 124 | return `ps -A`; 125 | default: 126 | return false; 127 | } 128 | })(); 129 | 130 | return new Promise((resolve, reject) => { 131 | require('child_process').exec(cmd, (err, stdout) => { 132 | if (err) reject(err); 133 | 134 | resolve(stdout.toLowerCase().indexOf(name.toLowerCase()) > -1); 135 | }); 136 | }); 137 | }; 138 | 139 | const watermarkUri = ({ uri, type } = {}) => { 140 | return uriParam({ uri, param: 'cml', value: type }); 141 | }; 142 | 143 | const preventcacheUri = ({ uri } = {}) => { 144 | return uriParam({ uri, param: 'cache-bypass', value: uuid.v4() }); 145 | }; 146 | 147 | const uriParam = (opts = {}) => { 148 | const { uri, param, value } = opts; 149 | const url = new URL(uri); 150 | url.searchParams.set(param, value); 151 | return url.toString(); 152 | }; 153 | 154 | const download = async (opts = {}) => { 155 | const { url, path } = opts; 156 | const res = await fetch(url, { agent: new ProxyAgent() }); 157 | const stream = fs.createWriteStream(path); 158 | return new Promise((resolve, reject) => { 159 | stream.on('error', (err) => reject(err)); 160 | res.body.pipe(stream); 161 | res.body.on('error', reject); 162 | stream.on('finish', resolve); 163 | }); 164 | }; 165 | 166 | const sshConnection = async (opts) => { 167 | const { host, username, privateKey, maxTries = 5 } = opts; 168 | 169 | const ssh = new NodeSSH(); 170 | 171 | let trials = 0; 172 | while (true) { 173 | try { 174 | await ssh.connect({ 175 | host, 176 | username, 177 | privateKey 178 | }); 179 | break; 180 | } catch (err) { 181 | if (maxTries === trials) throw err; 182 | trials += 1; 183 | await sleep(10); 184 | } 185 | } 186 | 187 | return ssh; 188 | }; 189 | 190 | const gpuPresent = async () => { 191 | let gpu = true; 192 | try { 193 | await exec('nvidia-smi'); 194 | } catch (err) { 195 | try { 196 | await exec('cuda-smi'); 197 | } catch (err) { 198 | gpu = false; 199 | } 200 | } 201 | 202 | return gpu; 203 | }; 204 | 205 | const tfCapture = async (command, args = [], options = {}) => { 206 | return new Promise((resolve, reject) => { 207 | const stderrCollection = []; 208 | const tfProc = require('child_process').spawn(command, args, options); 209 | tfProc.stdout.on('data', (buf) => { 210 | const parse = (line) => { 211 | if (line === '') return; 212 | try { 213 | const { '@level': level, '@message': message } = JSON.parse(line); 214 | if (level === 'error') { 215 | logger.error(`terraform error: ${message}`); 216 | } else { 217 | logger.info(message); 218 | } 219 | } catch (err) { 220 | logger.info(line); 221 | } 222 | }; 223 | buf.toString('utf8').split('\n').forEach(parse); 224 | }); 225 | tfProc.stderr.on('data', (buf) => { 226 | stderrCollection.push(buf); 227 | }); 228 | tfProc.on('close', (code) => { 229 | if (code !== 0) { 230 | const stderrOutput = Buffer.concat(stderrCollection).toString('utf8'); 231 | reject(stderrOutput); 232 | } 233 | resolve(); 234 | }); 235 | }); 236 | }; 237 | 238 | const fileExists = (path) => 239 | fs.promises.stat(path).then( 240 | () => true, 241 | () => false 242 | ); 243 | 244 | exports.tfCapture = tfCapture; 245 | exports.waitForever = waitForever; 246 | exports.exec = exec; 247 | exports.fetchUploadData = fetchUploadData; 248 | exports.upload = upload; 249 | exports.randid = randid; 250 | exports.sleep = sleep; 251 | exports.isProcRunning = isProcRunning; 252 | exports.watermarkUri = watermarkUri; 253 | exports.preventcacheUri = preventcacheUri; 254 | exports.download = download; 255 | exports.sshConnection = sshConnection; 256 | exports.gpuPresent = gpuPresent; 257 | exports.fileExists = fileExists; 258 | exports.getos = getos; 259 | -------------------------------------------------------------------------------- /src/utils.test.js: -------------------------------------------------------------------------------- 1 | const { exec, upload } = require('./utils'); 2 | 3 | describe('exec tests', () => { 4 | test('exec is await and outputs hello', async () => { 5 | const output = await exec('echo', 'hello'); 6 | expect(output).toMatch('hello'); 7 | }); 8 | 9 | test('Command rejects if failure', async () => { 10 | let error; 11 | try { 12 | await exec('this_command_fails'); 13 | } catch (err) { 14 | error = err; 15 | } 16 | 17 | expect(error).not.toBeNull(); 18 | }); 19 | }); 20 | 21 | describe('upload tests', () => { 22 | test('image/png', async () => { 23 | const { mime } = await upload({ path: 'assets/logo.png' }); 24 | expect(mime).toBe('image/png'); 25 | }); 26 | 27 | test('application/pdf', async () => { 28 | const { mime } = await upload({ path: 'assets/logo.pdf' }); 29 | expect(mime).toBe('application/pdf'); 30 | }); 31 | 32 | test('image/svg+xml', async () => { 33 | const { mime } = await upload({ path: 'assets/test.svg' }); 34 | expect(mime).toBe('image/svg+xml'); 35 | }); 36 | }); 37 | -------------------------------------------------------------------------------- /src/watermark.js: -------------------------------------------------------------------------------- 1 | const WATERMARK_IMAGE = 'https://cml.dev/watermark.png'; 2 | 3 | class Watermark { 4 | constructor(opts = {}) { 5 | const { label = '', workflow, run, sha = false } = opts; 6 | // Replace {workflow} and {run} placeholders in label with actual values. 7 | this.label = label.replace('{workflow}', workflow).replace('{run}', run); 8 | 9 | this.sha = sha; 10 | } 11 | 12 | // Appends the watermark (in markdown) to the report. 13 | appendTo(report) { 14 | return `${report}\n\n${this.toString()}`; 15 | } 16 | 17 | // Returns whether the watermark is present in the specified text. 18 | // When checking for presence, the commit sha in the watermark is ignored. 19 | isIn(text) { 20 | const title = escapeRegExp(this.title); 21 | const url = escapeRegExp(this.url({ sha: false })); 22 | const pattern = `!\\[\\]\\(${url}#[0-9a-fA-F]+ "${title}"\\)`; 23 | const re = new RegExp(pattern); 24 | return re.test(text); 25 | } 26 | 27 | // String representation of the watermark. 28 | toString() { 29 | // Replace {workflow} and {run} placeholders in label with actual values. 30 | return `![](${this.url()} "${this.title}")`; 31 | } 32 | 33 | get title() { 34 | let title = `CML watermark ${this.label}`.trim(); 35 | // Github appears to escape underscores and asterisks in markdown content. 36 | // Without escaping them, the watermark content in comments retrieved 37 | // from github will not match the input. 38 | const patterns = [ 39 | [/_/g, '\\_'], // underscore 40 | [/\*/g, '\\*'], // asterisk 41 | [/\[/g, '\\['], // opening square bracket 42 | [/ label.replace(pattern[0], pattern[1]), 46 | title 47 | ); 48 | return title; 49 | } 50 | 51 | url(opts = {}) { 52 | const { sha = this.sha } = opts; 53 | const watermarkUrl = new URL(WATERMARK_IMAGE); 54 | if (sha) { 55 | watermarkUrl.hash = sha; 56 | } 57 | return watermarkUrl.toString(); 58 | } 59 | } 60 | 61 | function escapeRegExp(str) { 62 | return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); 63 | } 64 | 65 | module.exports = { Watermark }; 66 | -------------------------------------------------------------------------------- /src/watermark.test.js: -------------------------------------------------------------------------------- 1 | const { Watermark } = require('../src/watermark'); 2 | 3 | describe('watermark tests', () => { 4 | beforeEach(() => { 5 | jest.resetModules(); 6 | }); 7 | 8 | test('append watermark to report', async () => { 9 | const watermark = new Watermark({ 10 | workflow: 'workflow-id', 11 | run: 'run-id', 12 | sha: 'deadbeef' 13 | }); 14 | const report = watermark.appendTo('some report'); 15 | expect(report).toEqual( 16 | 'some report\n\n![](https://cml.dev/watermark.png#deadbeef "CML watermark")' 17 | ); 18 | }); 19 | 20 | test('append watermark with label to report', async () => { 21 | const watermark = new Watermark({ 22 | label: 'some-label', 23 | workflow: 'workflow-id', 24 | run: 'run-id', 25 | sha: 'deadbeef' 26 | }); 27 | const report = watermark.appendTo('some report'); 28 | expect(report).toEqual( 29 | 'some report\n\n![](https://cml.dev/watermark.png#deadbeef "CML watermark some-label")' 30 | ); 31 | }); 32 | 33 | test('append watermark with workflow placeholder to report', async () => { 34 | const watermark = new Watermark({ 35 | label: 'some-label-{workflow}', 36 | workflow: 'workflow-id', 37 | run: 'run-id', 38 | sha: 'deadbeef' 39 | }); 40 | const report = watermark.appendTo('some report'); 41 | expect(report).toEqual( 42 | 'some report\n\n![](https://cml.dev/watermark.png#deadbeef "CML watermark some-label-workflow-id")' 43 | ); 44 | }); 45 | 46 | test('append watermark with run placeholder to report', async () => { 47 | const watermark = new Watermark({ 48 | label: 'some-label-{run}', 49 | workflow: 'workflow-id', 50 | run: 'run-id', 51 | sha: 'deadbeef' 52 | }); 53 | const report = watermark.appendTo('some report'); 54 | expect(report).toEqual( 55 | 'some report\n\n![](https://cml.dev/watermark.png#deadbeef "CML watermark some-label-run-id")' 56 | ); 57 | }); 58 | 59 | test('check for presence of the watermark in a report', async () => { 60 | const watermark = new Watermark({ 61 | workflow: 'workflow-id', 62 | run: 'run-id', 63 | sha: 'deadbeef' 64 | }); 65 | const report = watermark.appendTo('some report'); 66 | expect(watermark.isIn(report)).toEqual(true); 67 | }); 68 | 69 | test('check for presence of the watermark with special chars in a report', async () => { 70 | const watermark = new Watermark({ 71 | label: 'custom_1[*]-vm', 72 | workflow: 'workflow-id', 73 | run: 'run-id', 74 | sha: 'deadbeef' 75 | }); 76 | const report = watermark.appendTo('some report'); 77 | expect(watermark.isIn(report)).toEqual(true); 78 | }); 79 | 80 | test('check for presence of the watermark in a report, different sha', async () => { 81 | const watermark = new Watermark({ 82 | label: 'some-label-{run}', 83 | run: 'run-id', 84 | sha: 'deadbeef' 85 | }); 86 | const report = watermark.appendTo('some report'); 87 | 88 | // Watermark with a different sha, created by a CI run on 89 | // a different HEAD commit. 90 | const newWatermark = new Watermark({ 91 | label: 'some-label-{run}', 92 | run: 'run-id', 93 | sha: 'abcd' 94 | }); 95 | 96 | expect(newWatermark.isIn(report)).toEqual(true); 97 | }); 98 | 99 | test('check for watermark mismatch', async () => { 100 | const watermark = new Watermark({ 101 | label: 'some-label-{workflow}', 102 | workflow: 'workflow-id', 103 | sha: 'deadbeef' 104 | }); 105 | const report = watermark.appendTo('some report'); 106 | 107 | // Watermark with a different sha and different 108 | // workflow. 109 | const newWatermark = new Watermark({ 110 | workflow: 'different-workflow-id', 111 | label: 'some-label-{workflow}', 112 | run: 'run-id', 113 | sha: 'abcd' 114 | }); 115 | 116 | expect(newWatermark.isIn(report)).toEqual(false); 117 | }); 118 | }); 119 | -------------------------------------------------------------------------------- /tests/proxy.js: -------------------------------------------------------------------------------- 1 | const ProxyServer = require('transparent-proxy'); 2 | 3 | const PORT = 9093; 4 | const startProxy = () => { 5 | if (global.proxyTestsServer) return; 6 | 7 | global.proxyTestsServer = new ProxyServer(); 8 | global.proxyTestsServer.listen(PORT, '0.0.0.0', () => 9 | console.log(`Proxy listening on port ${PORT}`) 10 | ); 11 | }; 12 | const stopProxy = () => { 13 | console.log('Teardown Jest. Stoping Proxy...'); 14 | global.proxyTestsServer.close(); 15 | global.proxyTestsServer.unref(); 16 | }; 17 | 18 | module.exports = { 19 | startProxy, 20 | stopProxy, 21 | PORT 22 | }; 23 | -------------------------------------------------------------------------------- /tests/setup.js: -------------------------------------------------------------------------------- 1 | const { startProxy, PORT } = require('./proxy'); 2 | module.exports = () => { 3 | process.env.http_proxy = `http://localhost:${PORT}`; 4 | startProxy(); 5 | }; 6 | -------------------------------------------------------------------------------- /tests/teardown.js: -------------------------------------------------------------------------------- 1 | const { stopProxy } = require('./proxy'); 2 | module.exports = () => { 3 | stopProxy(); 4 | }; 5 | --------------------------------------------------------------------------------