├── .buildkite ├── Dockerfile ├── docker-compose.yml ├── pipeline.release.yml ├── pipeline.yml └── steps │ ├── prepare-release.sh │ ├── publish-pacts.sh │ ├── release.sh │ ├── tests.sh │ └── upload-linux-packages.sh ├── .github ├── CODEOWNERS ├── dependabot.yml └── pull_request_template.md ├── .gitignore ├── .golangci.yaml ├── .goreleaser.yaml ├── CHANGELOG.md ├── LICENSE.txt ├── README.md ├── SECURITY.md ├── bin ├── e2e ├── pact-record-support-ended ├── publish-pact ├── release-pact-version └── setup ├── docs ├── cypress.md ├── gotest.md ├── jest.md ├── playwright.md ├── pytest-pants.md ├── pytest.md └── rspec.md ├── go.mod ├── go.sum ├── internal ├── api │ ├── client.go │ ├── client_test.go │ ├── create_test_plan.go │ ├── create_test_plan_test.go │ ├── doc.go │ ├── fetch_files_timing.go │ ├── fetch_files_timing_test.go │ ├── fetch_test_plan.go │ ├── fetch_test_plan_test.go │ ├── filter_tests.go │ ├── filter_tests_test.go │ ├── post_test_plan_metadata.go │ └── post_test_plan_metadata_test.go ├── config │ ├── config.go │ ├── config_test.go │ ├── doc.go │ ├── env.go │ ├── env_test.go │ ├── error.go │ ├── read.go │ ├── read_test.go │ ├── validate.go │ └── validate_test.go ├── debug │ ├── debug.go │ ├── debug_test.go │ └── doc.go ├── env │ ├── env.go │ └── env_test.go ├── plan │ ├── doc.go │ ├── fallback.go │ ├── fallback_test.go │ └── type.go ├── runner │ ├── command.go │ ├── command_test.go │ ├── cypress.go │ ├── cypress_test.go │ ├── detector.go │ ├── discover.go │ ├── discover_test.go │ ├── doc.go │ ├── error.go │ ├── gotest.go │ ├── gotest_junit.go │ ├── gotest_junit_test.go │ ├── gotest_test.go │ ├── jest.go │ ├── jest_test.go │ ├── playwright.go │ ├── playwright_test.go │ ├── pytest.go │ ├── pytest_pants.go │ ├── pytest_pants_test.go │ ├── pytest_test.go │ ├── rspec.go │ ├── rspec_test.go │ ├── run_result.go │ ├── run_result_test.go │ ├── signal_unix.go │ ├── signal_windows.go │ ├── test_result.go │ ├── testdata │ │ ├── cypress │ │ │ ├── cypress.config.js │ │ │ ├── cypress │ │ │ │ └── e2e │ │ │ │ │ ├── failing_spec.cy.js │ │ │ │ │ ├── flaky_spec.cy.js │ │ │ │ │ └── passing_spec.cy.js │ │ │ ├── index.html │ │ │ └── package.json │ │ ├── files │ │ │ ├── animals │ │ │ │ ├── ant_test │ │ │ │ └── bee_test │ │ │ ├── fruits │ │ │ │ ├── apple_test │ │ │ │ └── banana_test │ │ │ └── vegetable_test │ │ ├── go │ │ │ ├── bad │ │ │ │ └── bad_test.go │ │ │ ├── go.mod │ │ │ └── hello_test.go │ │ ├── jest │ │ │ ├── failure.spec.js │ │ │ ├── jest.config.js │ │ │ ├── package.json │ │ │ ├── runtimeError.spec.js │ │ │ ├── skipped.spec.js │ │ │ ├── slow.spec.js │ │ │ └── spells │ │ │ │ └── expelliarmus.spec.js │ │ ├── package.json │ │ ├── playwright │ │ │ ├── .gitignore │ │ │ ├── index.html │ │ │ ├── package.json │ │ │ ├── playwright.config.js │ │ │ └── tests │ │ │ │ ├── error.spec.js │ │ │ │ ├── example.spec.js │ │ │ │ ├── failed.spec.js │ │ │ │ └── skipped.spec.js │ │ ├── pytest │ │ │ ├── .gitignore │ │ │ ├── failed_test.py │ │ │ ├── pytest-collector-result.json │ │ │ ├── result-failed.json │ │ │ ├── result-passed.json │ │ │ └── test_sample.py │ │ ├── pytest_pants │ │ │ ├── .gitignore │ │ │ ├── 3rdparty │ │ │ │ └── python │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── pytest-requirements.txt │ │ │ │ │ └── pytest.lock │ │ │ ├── BUILD │ │ │ ├── README.md │ │ │ ├── failing_test.py │ │ │ ├── pants.toml │ │ │ ├── passing_test.py │ │ │ └── result-failed.json │ │ ├── rspec │ │ │ ├── Gemfile │ │ │ ├── Gemfile.lock │ │ │ └── spec │ │ │ │ ├── bad_syntax_spec.rb │ │ │ │ ├── failure_spec.rb │ │ │ │ ├── shared_examples.rb │ │ │ │ ├── skipped_spec.rb │ │ │ │ ├── specs_with_shared_examples_spec.rb │ │ │ │ └── spells │ │ │ │ └── expelliarmus_spec.rb │ │ ├── segv.sh │ │ └── yarn.lock │ └── util_test.go └── version │ └── version.go ├── main.go ├── main_test.go ├── packaging └── Dockerfile └── testdata ├── retry.sh └── rspec └── spec ├── bad_syntax_spec.rb └── fruits ├── apple_spec.rb ├── banana_spec.rb ├── fig_spec.rb └── tomato_spec.rb /.buildkite/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:3.4.4-slim-bookworm AS ruby 2 | FROM cypress/included:14.4.1 AS cypress 3 | FROM python:3.13.5-bookworm AS python 4 | 5 | FROM golang:1.24.4-bookworm AS golang 6 | 7 | COPY --from=ruby / / 8 | COPY --from=cypress / / 9 | COPY --from=python / / 10 | 11 | RUN gem install rspec 12 | RUN yarn global add jest 13 | RUN pip install pytest 14 | RUN pip install buildkite-test-collector==0.2.0 15 | RUN curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash -s -- --bin-dir /usr/local/bin 16 | 17 | # Install curl, download bktec binary, make it executable, place it, and cleanup 18 | RUN apt-get update && \ 19 | apt-get install -y --no-install-recommends curl && \ 20 | echo "Downloading bktec..." && \ 21 | curl -L -o /usr/local/bin/bktec "https://github.com/buildkite/test-engine-client/releases/download/v1.5.0-rc.1/bktec_1.5.0-rc.1_linux_amd64" && \ 22 | echo "Setting execute permissions..." && \ 23 | chmod +x /usr/local/bin/bktec && \ 24 | echo "bktec installed successfully:" && \ 25 | bktec --version 26 | -------------------------------------------------------------------------------- /.buildkite/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | 3 | services: 4 | ci: 5 | build: 6 | context: . 7 | dockerfile: Dockerfile 8 | volumes: 9 | - ../:/work:cached 10 | - ~/gocache:/gocache 11 | - ~/gomodcache:/gomodcache 12 | working_dir: /work 13 | environment: 14 | - CI 15 | - BUILDKITE_JOB_ID 16 | - GOCACHE=/gocache 17 | - GOMODCACHE=/gomodcache 18 | -------------------------------------------------------------------------------- /.buildkite/pipeline.release.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: ":cook: Prepare release" 3 | plugins: 4 | - docker#v5.11.0: 5 | image: ghcr.io/caarlos0/svu:v2.1.0 6 | entrypoint: "" 7 | command: [".buildkite/steps/prepare-release.sh"] 8 | mount-buildkite-agent: true 9 | 10 | - wait 11 | 12 | - name: ":rocket: Release" 13 | artifact_paths: "dist/**/*" 14 | plugins: 15 | - aws-assume-role-with-web-identity: 16 | role-arn: arn:aws:iam::445615400570:role/pipeline-buildkite-test-engine-client-release 17 | - aws-ssm#v1.0.0: 18 | parameters: 19 | GITHUB_TOKEN: /pipelines/buildkite/test-engine-client-release/GH_TOKEN 20 | DOCKERHUB_USER: /pipelines/buildkite/test-engine-client-release/dockerhub-user 21 | DOCKERHUB_PASSWORD: /pipelines/buildkite/test-engine-client-release/dockerhub-password 22 | - docker#v5.11.0: 23 | image: goreleaser/goreleaser:v2.8.1 24 | entrypoint: "" 25 | command: [".buildkite/steps/release.sh"] 26 | mount-buildkite-agent: true 27 | volumes: 28 | - "/var/run/docker.sock:/var/run/docker.sock" 29 | environment: 30 | - GITHUB_TOKEN 31 | - DOCKERHUB_USER 32 | - DOCKERHUB_PASSWORD 33 | -------------------------------------------------------------------------------- /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | env: 2 | AWS_DEFAULT_REGION: us-east-1 3 | 4 | steps: 5 | - name: ":golangci-lint: Lint" 6 | plugins: 7 | - docker#v5.11.0: 8 | image: golangci/golangci-lint:v1.64.8 9 | workdir: /go/src/github.com/your-org/your-repo 10 | command: 11 | - golangci-lint 12 | - run 13 | 14 | - name: ":go: Tests" 15 | command: ".buildkite/steps/tests.sh" 16 | parallelism: 2 17 | artifact_paths: 18 | - cover.{html,out} 19 | - internal/api/pacts/* 20 | plugins: 21 | - aws-assume-role-with-web-identity: 22 | role-arn: arn:aws:iam::445615400570:role/pipeline-buildkite-test-engine-client 23 | - aws-ssm#v1.0.0: 24 | parameters: 25 | BUILDKITE_TEST_ENGINE_SUITE_TOKEN: /pipelines/buildkite/test-engine-client/SUITE_TOKEN 26 | BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN: /pipelines/buildkite/test-engine-client/API_ACCESS_TOKEN 27 | - docker-compose#v4.14.0: 28 | config: .buildkite/docker-compose.yml 29 | cli-version: 2 30 | run: ci 31 | propagate-environment: true 32 | environment: 33 | - BUILDKITE_TEST_ENGINE_SUITE_TOKEN 34 | - BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN 35 | - test-collector#v1.11.0: 36 | files: "junit-*.xml" 37 | format: "junit" 38 | # This is to prevent the test runner as part of our test fixture trigger test collection 39 | # such as pytest + python test collector 40 | api-token-env-name: "BUILDKITE_TEST_ENGINE_SUITE_TOKEN" 41 | 42 | - wait 43 | 44 | - group: ":hammer_and_wrench: Build binaries" 45 | steps: 46 | - name: ":{{matrix.os}}: Build {{matrix.os}} {{matrix.arch}} binary" 47 | artifact_paths: "dist/**/*" 48 | plugins: 49 | docker#v5.11.0: 50 | image: goreleaser/goreleaser:v2.8.1 51 | mount-buildkite-agent: true 52 | environment: 53 | - GOOS={{matrix.os}} 54 | - GOARCH={{matrix.arch}} 55 | command: 56 | - build 57 | - --single-target 58 | - --snapshot 59 | matrix: 60 | setup: 61 | os: 62 | - darwin 63 | - linux 64 | - windows 65 | arch: 66 | - amd64 67 | - arm64 68 | -------------------------------------------------------------------------------- /.buildkite/steps/prepare-release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | current=$(svu current) 4 | 5 | 6 | options() { 7 | if [[ "$current" =~ "-rc" ]]; then 8 | # if current version is 0.7.5-rc.1 9 | # the options should be 10 | # - 0.7.5 11 | # - 0.7.5-rc.2 12 | cat < 7 | 8 | ### Context 9 | 10 | 13 | 14 | ### Changes 15 | 16 | 21 | 22 | ### Testing 23 | 24 | 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | 23 | # Pact generated files 24 | internal/api/pacts/ 25 | 26 | dist/ 27 | node_modules/ 28 | 29 | internal/runner/testdata/jest/jest-result.json 30 | 31 | internal/runner/testdata/rspec/rspec-result.json 32 | 33 | .envrc 34 | 35 | .direnv 36 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | linters: 2 | enable: 3 | - gosec 4 | 5 | issues: 6 | exclude-rules: 7 | - path: _test.go 8 | linters: 9 | - errcheck 10 | - gosec 11 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | project_name: test-engine-client 3 | 4 | release: 5 | name_template: Test Engine Client v{{.Version}} 6 | draft: false 7 | prerelease: auto 8 | make_latest: "{{not .Prerelease}}" 9 | mode: replace 10 | 11 | changelog: 12 | use: github-native 13 | 14 | archives: 15 | - format: binary 16 | 17 | builds: 18 | - env: 19 | - CGO_ENABLED=0 20 | goos: [linux, darwin] 21 | goarch: [amd64, arm64] 22 | ldflags: "-X 'github.com/buildkite/test-engine-client/internal/version.Version=v{{ .Version }}'" 23 | binary: bktec 24 | 25 | checksum: 26 | name_template: "bktec_{{ .Version }}_checksums.txt" 27 | 28 | brews: 29 | - name: bktec 30 | description: "Buildkite Test Engine Client" 31 | homepage: "https://github.com/buildkite/test-engine-client" 32 | skip_upload: auto 33 | directory: . 34 | test: | 35 | version_output = shell_output("bktec --version") 36 | assert_match "v#{version}\n", version_output 37 | repository: 38 | owner: buildkite 39 | name: homebrew-buildkite 40 | branch: master 41 | 42 | git: 43 | ignore_tags: 44 | - '{{ envOrDefault "GORELEASER_IGNORE_TAG" ""}}' 45 | 46 | dockers: 47 | - image_templates: 48 | - "packages.buildkite.com/buildkite/test-engine-client-docker/test-engine-client:v{{ .Version }}-amd64" 49 | dockerfile: "packaging/Dockerfile" 50 | build_flag_templates: 51 | - "--platform=linux/amd64" 52 | - image_templates: 53 | - "packages.buildkite.com/buildkite/test-engine-client-docker/test-engine-client:v{{ .Version }}-arm64" 54 | goarch: arm64 55 | dockerfile: "packaging/Dockerfile" 56 | build_flag_templates: 57 | - "--platform=linux/arm64" 58 | - image_templates: 59 | - "buildkite/test-engine-client:v{{ .Version }}-amd64" 60 | # skip pushing image to Dockerhub if it's a prerelease 61 | skip_push: auto 62 | dockerfile: "packaging/Dockerfile" 63 | build_flag_templates: 64 | - "--platform=linux/amd64" 65 | - image_templates: 66 | - "buildkite/test-engine-client:v{{ .Version }}-arm64" 67 | # skip pushing image to Dockerhub if it's a prerelease 68 | skip_push: auto 69 | goarch: arm64 70 | dockerfile: "packaging/Dockerfile" 71 | build_flag_templates: 72 | - "--platform=linux/arm64" 73 | docker_manifests: 74 | - name_template: "packages.buildkite.com/buildkite/test-engine-client-docker/test-engine-client:v{{ .Version }}" 75 | image_templates: 76 | - "packages.buildkite.com/buildkite/test-engine-client-docker/test-engine-client:v{{ .Version }}-amd64" 77 | - "packages.buildkite.com/buildkite/test-engine-client-docker/test-engine-client:v{{ .Version }}-arm64" 78 | - name_template: "packages.buildkite.com/buildkite/test-engine-client-docker/test-engine-client:latest" 79 | image_templates: 80 | - "packages.buildkite.com/buildkite/test-engine-client-docker/test-engine-client:v{{ .Version }}-amd64" 81 | - "packages.buildkite.com/buildkite/test-engine-client-docker/test-engine-client:v{{ .Version }}-arm64" 82 | - name_template: "buildkite/test-engine-client:v{{ .Version }}" 83 | image_templates: 84 | - "buildkite/test-engine-client:v{{ .Version }}-amd64" 85 | - "buildkite/test-engine-client:v{{ .Version }}-arm64" 86 | # skip pushing manifest to Dockerhub if it's a prerelease 87 | skip_push: auto 88 | - name_template: "buildkite/test-engine-client:latest" 89 | image_templates: 90 | - "buildkite/test-engine-client:v{{ .Version }}-amd64" 91 | - "buildkite/test-engine-client:v{{ .Version }}-arm64" 92 | # skip pushing manifest to Dockerhub if it's a prerelease 93 | skip_push: auto 94 | 95 | nfpms: 96 | - vendor: Buildkite 97 | id: linux-pkg 98 | package_name: bktec 99 | homepage: https://github.com/buildkite/test-engine-client 100 | maintainer: Buildkite 101 | description: Buildkite Test Engine Client 102 | license: MIT 103 | formats: 104 | - deb 105 | - rpm 106 | provides: 107 | - bktec 108 | 109 | publishers: 110 | - name: buildkite-packages 111 | disable: "{{if .Prerelease}}true{{end}}" 112 | cmd: .buildkite/steps/upload-linux-packages.sh {{ .ArtifactPath }} 113 | ids: 114 | - linux-pkg 115 | env: 116 | - BUILDKITE_JOB_ID={{ .Env.BUILDKITE_JOB_ID }} 117 | - BUILDKITE_AGENT_ACCESS_TOKEN={{ .Env.BUILDKITE_AGENT_ACCESS_TOKEN }} 118 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.5.0 - 2025-05-30 4 | - Add support for the Go test runner. 5 | - Change the retry behavior to automatically retry muted tests that fail. To disable this, set the `BUILDKITE_TEST_ENGINE_DISABLE_RETRY_FOR_MUTED_TEST` environment variable to `true`. 6 | 7 | ## 1.4.0 - 2025-02-14 8 | - Support pytest. 9 | - Upgrade Go to 1.24. 10 | - Filter jest command with file paths on retries 11 | 12 | ## 1.3.3 - 2025-02-14 13 | - Update server-side error handling. 14 | 15 | ## 1.3.2 - 2025-01-20 16 | - Fix issue where a test incorrectly reported as "Passed on Retry". 17 | 18 | ## 1.3.1 - 2025-01-10 19 | - Fix issue where non-RSpec runners would terminate when attempting to split by example, as splitting by example is only supported in RSpec. 20 | 21 | ## 1.3.0 - 2024-12-20 22 | - Add skipped tests to the test report. 23 | - Add support for muted tests in job retry. 24 | - Add run statistic to the test plan metadata. 25 | 26 | ## 1.2.1 - 2024-12-12 27 | - Fix issue where the run would pass despite errors outside of tests, such as syntax or runtime errors. 28 | 29 | ## 1.2.0 - 2024-11-26 30 | - Add support for muting tests. 31 | - Fix issue with Cypress command by passing the list of test files separated by commas. 32 | 33 | ## 1.1.0 - 2024-11-11 34 | - Experimental support for Cypress. See [Cypress usage guide](./docs/cypress.md). 35 | - Experimental support for Playwright. See [Playwright usage guide](./docs/playwright.md). 36 | - Update `BUILDKITE_TEST_ENGINE_TEST_CMD` and `BUILDKITE_TEST_ENGINE_RETRY_CMD` for Jest. See [Jest usage guide](./docs/jest.md). 37 | - Fix issue when retrying Jest tests with special characters 38 | - Remove `**/node_modules` from default value of `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN`. Files inside `node_modules` will be ignore regardless the value of this environment variable. 39 | 40 | ## 1.0.0 - 2024-09-23 41 | - ⚠️ **BREAKING** Rename all environment variables from `BUILDKITE_SPLITTER_*` to `BUILDKITE_TEST_ENGINE_*`. See [Migrating to 1.0.0](https://github.com/buildkite/test-splitter/tree/90b699918b11500336f8a0fce306da917fba7408?tab=readme-ov-file#migrating-to-100) 42 | - ⚠️ **BREAKING** Add the `BUILDKITE_TEST_ENGINE_TEST_RUNNER` as required environment variable. 43 | 44 | ## 0.9.1 - 2024-09-16 45 | - Fix issue with split by example when shared examples are used in RSpec 46 | 47 | ## 0.9.0 - 2024-09-11 48 | - ⚠️ **BREAKING** Add the `BUILDKITE_SPLITTER_RESULT_PATH` required environment variable. See [Migrating to 0.9.0](https://github.com/buildkite/test-splitter/tree/db4cab8cd6c82392553cd80481cf75e3888c2f4c?tab=readme-ov-file#migrating-to-090). 49 | - Experimental support for Jest by setting `BUILDKITE_SPLITTER_TEST_RUNNER` to `jest`. 50 | - Update the retry behavior to only retry failed tests. 51 | - Update split-by-example behavior to perform more work server-side. 52 | - Improve configuration error message. 53 | - Fix issue printing dry-run errors. 54 | - Fix issue with `BUILDKITE_STEP_ID` presence validation. 55 | 56 | ## 0.8.1 - 2024-08-06 57 | - Add `BUILDKITE_BRANCH` env var for test plan experiments 58 | - Fix to zzglob library issue where files not matching the include pattern are in the test plan 59 | 60 | ## 0.8.0 - 2024-07-26 61 | - Add support to customize the rspec retry command. 62 | - Fix issue with file globbing during the file discovery. 63 | 64 | ## 0.7.3 - 2024-07-19 65 | - Improve handling when the runner terminates due to an OS-level signal. 66 | 67 | ## 0.7.2 - 2024-07-03 68 | - Fix log statement newline issue. 69 | 70 | ## 0.7.1 - 2024-07-02 71 | - Fix issue where `--version` would fail if no environment configured. 72 | - Prefix log statements with 'Buildkite Test Splitter'. 73 | 74 | ## 0.7.0 - 2024-06-27 75 | - Remove the ability to override the test plan identifier via `BUILDKITE_SPLITTER_IDENTIFIER`. 76 | - Add support for orchestration page in Buildkite, by sending metadata after tests execution. 77 | 78 | ## 0.6.2 - 2024-06-24 79 | - Fix issue where the client version is not set in the user agent. 80 | 81 | ## 0.6.1 - 2024-06-21 82 | - Ignore request body when it is empty or when the request is a GET request. 83 | 84 | ## 0.6.0 - 2024-06-21 85 | 86 | - ⚠️ **BREAKING** Remove support for the undocumented `--files` flag. 87 | - ⚠️ **BREAKING** Rename the `BUILDKITE_API_ACCESS_TOKEN` environment variable to `BUILDKITE_SPLITTER_API_ACCESS_TOKEN`. 88 | - Add support for split-by-example using the `BUILDKITE_SPLITTER_SPLIT_BY_EXAMPLE` environment variable. 89 | - Add support for more verbose debug logging using the `BUILDKITE_SPLITTER_DEBUG_ENABLED` environment variable. 90 | 91 | ## 0.5.1 92 | - Add a new line to each error log. 93 | 94 | ## 0.5.0 95 | - ⚠️ **BREAKING** Rename `BUILDKITE_TEST_SPLITTER_CMD` to `BUILDKITE_SPLITTER_TEST_CMD`. 96 | - ⚠️ **BREAKING** Change the authentication mechanism to use Buildkite API access token. See [Migrating to 0.5.0](https://github.com/buildkite/test-splitter/tree/cdbbe348a0eb10bb6ca3211f2c5cd870f0dadfdd?tab=readme-ov-file#migrating-from-040). 97 | - Add support for automatically retrying failed tests using `BUILDKITE_SPLITTER_RETRY_COUNT`. 98 | - Add `--version` flag to aid in debugging. 99 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Buildkite Pty Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Buildkite Test Engine Client 2 | 3 | Buildkite Test Engine Client (bktec) is an open source tool to orchestrate your test suites. It uses your Buildkite Test Engine suite data to intelligently partition and parallelize your tests. 4 | 5 | bktec supports multiple test runners and offers various features to enhance your testing workflow. Below is a comparison of the features supported by each test runner: 6 | 7 | | Feature | RSpec | Jest | Playwright | Cypress | pytest | pants (pytest) | Go test | 8 | | -------------------------------------------------- | :---: | :--: | :---------: | :-----: | :-----: | :------------: | :-----: | 9 | | Filter test files | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | 10 | | Automatically retry failed test | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | 11 | | Split slow files by individual test example | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | 12 | | Mute tests (ignore test failures) | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | 13 | | Skip tests | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | 14 | 15 | ## Installation 16 | The latest version of bktec can be downloaded from https://github.com/buildkite/test-engine-client/releases 17 | 18 | ### Supported OS/Architecture 19 | ARM and AMD architecture for linux and darwin 20 | 21 | The available Go binaries 22 | - bktec-darwin-amd64 23 | - bktec-darwin-arm64 24 | - bktec-linux-amd64 25 | - bktec-linux-arm64 26 | 27 | ## Using bktec 28 | 29 | ### Buildkite Pipeline environment variables 30 | bktec uses the following Buildkite Pipeline provided environment variables. 31 | | Environment Variable | Description| 32 | | -------------------- | ----------- | 33 | | `BUILDKITE_BUILD_ID` | The UUID of the Buildkite build. bktec uses this UUID along with `BUILDKITE_STEP_ID` to uniquely identify the test plan. | 34 | | `BUILDKITE_JOB_ID` | The UUID of the job in Buildkite build. | 35 | | `BUILDKITE_ORGANIZATION_SLUG` | The slug of your Buildkite organization. | 36 | | `BUILDKITE_PARALLEL_JOB` | The index number of a parallel job created from a Buildkite parallel build step.
Make sure you configure `parallelism` in your pipeline definition. You can read more about Buildkite parallel build step on this [page](https://buildkite.com/docs/pipelines/controlling-concurrency#concurrency-and-parallelism).| 37 | | `BUILDKITE_PARALLEL_JOB_COUNT` | The total number of parallel jobs created from a Buildkite parallel build step.
Make sure you configure `parallelism` in your pipeline definition. You can read more about Buildkite parallel build step on this [page](https://buildkite.com/docs/pipelines/controlling-concurrency#concurrency-and-parallelism). | 38 | | `BUILDKITE_STEP_ID` | The UUID of the step group in Buildkite build. bktec uses this UUID along with `BUILDKITE_BUILD_ID` to uniquely identify the test plan. 39 | 40 | > [!IMPORTANT] 41 | > Please make sure that the above environment variables are available in your testing environment, particularly if you use Docker or some other type of containerization to run your tests. 42 | 43 | ### Create API access token 44 | To use bktec, you need a Buildkite API access token with `read_suites`, `read_test_plan`, and `write_test_plan` scopes. You can generate this token from your [Personal Settings](https://buildkite.com/user/api-access-tokens) in Buildkite. After creating the token, set the `BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN` environment variable with the token value. 45 | 46 | ```sh 47 | export BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN=token 48 | ``` 49 | 50 | ### Configure Test Engine suite slug 51 | To use bktec, you need to configure the `BUILDKITE_TEST_ENGINE_SUITE_SLUG` environment variable with your Test Engine suite slug. You can find the suite slug in the URL of your suite. For example, in the URL `https://buildkite.com/organizations/my-organization/analytics/suites/my-suite`, the slug is `my-suite`. 52 | 53 | ```sh 54 | export BUILDKITE_TEST_ENGINE_SUITE_SLUG=my-slug 55 | ``` 56 | 57 | ### Configure the test runner 58 | To configure the test runner for bktec, please refer to the detailed guides provided for each supported test runner. You can find the guides at the following links: 59 | - [Jest](./docs/jest.md) 60 | - [Playwright](./docs/playwright.md) 61 | - [Cypress](./docs/cypress.md) 62 | - [pytest](./docs/pytest.md) 63 | - [pytest pants](./docs/pytest-pants.md) 64 | - [go test](./docs/gotest.md) 65 | - [RSpec](./docs/rspec.md) 66 | 67 | 68 | ### Running bktec 69 | Please download the executable and make it available in your testing environment. 70 | To parallelize your tests in your Buildkite build, you can amend your pipeline step configuration to: 71 | ``` 72 | steps: 73 | - name: "Rspec" 74 | command: ./bktec 75 | parallelism: 10 76 | env: 77 | BUILDKITE_TEST_ENGINE_SUITE_SLUG: my-suite 78 | BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN: your-secret-token 79 | BUILDKITE_TEST_ENGINE_TEST_RUNNER: rspec 80 | BUILDKITE_TEST_ENGINE_RESULT_PATH: tmp/result.json 81 | ``` 82 | 83 | > [!TIP] 84 | > You can find example configurations and usage instructions for each test runner in our [examples repository](https://github.com/buildkite/test-engine-client-examples). 85 | 86 | 87 | ### Debugging 88 | To enable debug mode, set the `BUILDKITE_TEST_ENGINE_DEBUG_ENABLED` environment variable to `true`. This will print detailed output to assist in debugging bktec. 89 | 90 | ### Possible exit statuses 91 | 92 | bktec may exit with a variety of exit statuses, outlined below: 93 | 94 | - If there is a configuration error, bktec will exit with 95 | status 16. 96 | - If the test runner (e.g. RSpec) exits cleanly, the exit status of 97 | the runner is returned. This will likely be 0 for successful test runs, 1 for 98 | failing test runs, but may be any other error status returned by the runner. 99 | - If the test runner is terminated by an OS level signal, such as SIGSEGV or 100 | SIGABRT, the exit status returned will be equal to 128 plus the signal number. 101 | For example, if the runner raises a SIGSEGV, the exit status will be (128 + 102 | 11) = 139. 103 | 104 | ## Development 105 | 106 | Make sure you have Go, Ruby, and Node.js installed in your environment. You can follow the installation guides for each of these tools: 107 | 108 | - [Go Installation Guide](https://golang.org/doc/install) 109 | - [Ruby Installation Guide](https://www.ruby-lang.org/en/documentation/installation/) 110 | - [Node.js Installation Guide](https://nodejs.org/en/download/package-manager/) 111 | 112 | Once you have these dependencies installed, run `bin/setup` to install dependencies for the sample projects for testing purposes. 113 | 114 | To test, run: 115 | ```sh 116 | go test ./... 117 | ``` 118 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Security 2 | 3 | Buildkite takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Buildkite](https://github.com/buildkite), [Bazelkite](https://github.com/bazelkite), [Buildbox](https://github.com/buildbox) and [Packagecloud](https://github.com/computology). 4 | 5 | If you believe you have found a security vulnerability in any Buildkite repository, please report it to us as described below. 6 | 7 | ## Reporting Security Issues 8 | 9 | **Please do not report security vulnerabilities through public GitHub issues.** 10 | 11 | Instead, please report them via the [HackerOne Bug Bounty program](https://hackerone.com/buildkite?type=team). Currently, the program is private and invite only. You can request access by contacting us [directly](mailto:security@buildkite.com?subject=Vulnerability reporting invite). 12 | 13 | If you prefer to submit without logging in, you can email us at [security@buildkite.com](mailto:security@buildkite.com). If possible, please encrypt your message with our PGP key. You can download it from the [Buildkite Security](https://buildkite.com/security) page. 14 | 15 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 16 | 17 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 18 | * Full paths of source file(s) related to the manifestation of the issue 19 | * The location of the affected source code (tag/branch/commit or direct URL) 20 | * Any special configuration required to reproduce the issue 21 | * Step-by-step instructions to reproduce the issue 22 | * Proof-of-concept or exploit code (if possible) 23 | * Impact of the issue, including how an attacker might exploit the issue 24 | 25 | This information will help us triage your report more quickly. 26 | 27 | ## Preferred Languages 28 | 29 | We prefer all communications to be in English. 30 | 31 | ## Policy 32 | 33 | We reserve the right to update, modify or replace this policy. If you have suggestions on how we could improve this process, please reach out to us at [support@buildkite.com](mailto:support@buildkite.com). 34 | -------------------------------------------------------------------------------- /bin/e2e: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script is used to run bktec against the sample project for the given test runner. 4 | # Sample project can be found in internal/runner/testdata/ 5 | # 6 | # Usage: ./bin/e2e 7 | # 8 | # Note: you need to manually set the following environment variables 9 | # - BUILDKITE_ORGANIZATION_SLUG 10 | # - BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN 11 | # - BUILDKITE_TEST_ENGINE_SUITE_SLUG 12 | 13 | export BUILDKITE_TEST_ENGINE_TEST_RUNNER=${1:-rspec} 14 | export BUILDKITE_TEST_ENGINE_RESULT_PATH="${BUILDKITE_TEST_ENGINE_TEST_RUNNER}-result.json" 15 | 16 | export BUILDKITE_BUILD_ID=$(date +%s) 17 | export BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-0} 18 | export BUILDKITE_PARALLEL_JOB_COUNT=${BUILDKITE_PARALLEL_JOB_COUNT:-2} 19 | export BUILDKITE_STEP_ID=$BUILDKITE_TEST_ENGINE_TEST_RUNNER 20 | 21 | # Override the following variables to the default value, in case they are set somewhere else 22 | export BUILDKITE_TEST_ENGINE_TEST_CMD="" 23 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN="" 24 | 25 | # Extra configuration for playwright 26 | if [ "$BUILDKITE_TEST_ENGINE_TEST_RUNNER" == "playwright" ]; then 27 | # We need to tell bktec to use playwright's result path configured in playwright.config.js 28 | export BUILDKITE_TEST_ENGINE_RESULT_PATH="test-results/results.json" 29 | # error.spec.js will prevent other tests from running, so we exclude it 30 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN="**/*/error.spec.js" 31 | fi 32 | 33 | cd ./internal/runner/testdata/$BUILDKITE_TEST_ENGINE_TEST_RUNNER 34 | 35 | go run ../../../../main.go 36 | -------------------------------------------------------------------------------- /bin/pact-record-support-ended: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Use this when there are no longer any customers using a particular version of 4 | # bktec. This will stop bk/bk from verifying against the pact of the unused 5 | # version. You'll need to have an environment variable called PACT_BROKER_TOKEN 6 | # with a write-access token for pact flow to do the business. 7 | # 8 | # Run it like: bin/pact-record-support-ended v0.6.0 9 | 10 | set -euo pipefail 11 | 12 | VERSION=$1 13 | 14 | ENVIRONMENT=production 15 | BROKER_BASE_URL=https://buildkite.pactflow.io 16 | PACTICIPANT=TestSplitterClient 17 | 18 | docker run \ 19 | --rm \ 20 | -it \ 21 | -e PACT_BROKER_TOKEN \ 22 | pactfoundation/pact-cli \ 23 | pact-broker \ 24 | record-support-ended \ 25 | --environment=${ENVIRONMENT} \ 26 | --pacticipant=${PACTICIPANT} \ 27 | --broker-base-url=${BROKER_BASE_URL} \ 28 | --version=${VERSION} 29 | -------------------------------------------------------------------------------- /bin/publish-pact: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Use this to publish pact json files to the pact broker. 4 | # The pact JSON needs to be generated first by running the tests. 5 | # You'll need to have an environment variable called PACT_BROKER_TOKEN 6 | # with a write-access token. 7 | # 8 | # Run it like: bin/publish-pact v0.6.0 9 | 10 | VERSION=$1 11 | PACT_BROKER_BASE_URL=https://buildkite.pactflow.io 12 | 13 | docker run --rm \ 14 | -w ${PWD} \ 15 | -v ${PWD}:${PWD} \ 16 | -e PACT_BROKER_BASE_URL \ 17 | -e PACT_BROKER_TOKEN \ 18 | pactfoundation/pact-cli:latest \ 19 | publish \ 20 | ${PWD}/internal/api/pacts \ 21 | --consumer-app-version ${VERSION} \ 22 | --tag-with-git-branch 23 | -------------------------------------------------------------------------------- /bin/release-pact-version: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Use this to record a published pact version as a production release. 4 | # You'll need to have an environment variable called PACT_BROKER_TOKEN 5 | # with a write-access token. 6 | # 7 | # Run it like: bin/release-pact-version v0.6.0 8 | 9 | VERSION=$1 10 | PACT_BROKER_BASE_URL=https://buildkite.pactflow.io 11 | 12 | docker run --rm \ 13 | -e PACT_BROKER_BASE_URL \ 14 | -e PACT_BROKER_TOKEN \ 15 | pactfoundation/pact-cli:latest \ 16 | pact-broker \ 17 | record-release \ 18 | --environment production \ 19 | --pacticipant TestEngineClient \ 20 | --version ${VERSION} 21 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # install pact-go as a dev dependency 4 | go get github.com/pact-foundation/pact-go/v2 5 | 6 | # install the `pact-go` CLI 7 | go install github.com/pact-foundation/pact-go/v2 8 | 9 | # Check if asdf is installed and being used for Go 10 | if command -v asdf &> /dev/null && asdf current golang &> /dev/null; then 11 | echo "🔄 Reshimming asdf golang..." 12 | asdf reshim golang 13 | fi 14 | 15 | # download and install the required libraries. 16 | # TODO if pact-go check return non- zero then install it 17 | if ! pact-go check &> /dev/null; then 18 | echo "🔄 Installing pact-go dependencies..." 19 | sudo pact-go -l DEBUG install 20 | else 21 | echo "✅ pact-go dependencies already installed" 22 | fi 23 | 24 | echo "🛠️ Installing dependencies for sample projects..." 25 | 26 | cd ./internal/runner/testdata 27 | # if yarn is available, use it to install dependencies 28 | # otherwise, use npm 29 | if command -v yarn &> /dev/null 30 | then 31 | yarn install 32 | else 33 | npm install 34 | fi 35 | 36 | # Install Playwright dependencies 37 | cd ./playwright 38 | npx playwright install 39 | npx playwright install-deps 40 | 41 | # Install Cypress dependencies 42 | cd ../cypress 43 | npx cypress install 44 | npx cypress verify 45 | 46 | # Install RSpec dependencies 47 | cd ../rspec 48 | bundle install 49 | 50 | # Install various python things, dependencies for pytest test cases 51 | if [ -n "$VIRTUAL_ENV" ]; then 52 | echo "Python virtual environment is active: $VIRTUAL_ENV" 53 | else 54 | echo "No python virtual environment active, creating .venv..." 55 | echo "You may have to activate the venv by yourself to make it work!" 56 | echo " source .venv/bin/activate" 57 | python -m venv .venv && source .venv/bin/activate 58 | fi 59 | pip install pytest 60 | pip install buildkite-test-collector==0.2.0 61 | 62 | echo "💖 Everything is fantastic!" 63 | -------------------------------------------------------------------------------- /docs/cypress.md: -------------------------------------------------------------------------------- 1 | # Using bktec with Cypress 2 | To integrate bktec with Cypress, set the `BUILDKITE_TEST_ENGINE_TEST_RUNNER` environment variable to `cypress`. 3 | 4 | ```sh 5 | export BUILDKITE_TEST_ENGINE_TEST_RUNNER=cypress 6 | ``` 7 | 8 | ## Configure test command 9 | By default, bktec runs Cypress with the following command: 10 | 11 | ```sh 12 | npx cypress run --spec {{testExamples}} 13 | ``` 14 | 15 | In this command, `{{testExamples}}` is replaced by bktec with the list of test files to run. You can customize this command using the `BUILDKITE_TEST_ENGINE_TEST_CMD` environment variable. 16 | 17 | To customize the test command, set the following environment variable: 18 | ```sh 19 | export BUILDKITE_TEST_ENGINE_TEST_CMD="yarn cypress:run --spec {{testExamples}}" 20 | ``` 21 | 22 | ## Filter test files 23 | By default, bktec runs test files that match the `**/*.cy.{js,jsx,ts,tsx}` pattern. You can customize this pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` environment variable. For instance, to configure bktec to only run Cypress test files inside a `cypress/e2e` directory, use: 24 | ```sh 25 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=cypress/e2e/**/*.cy.js 26 | ``` 27 | 28 | Additionally, you can exclude specific files or directories that match a certain pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` environment variable. For example, to exclude test files inside the `cypress/component` directory, use: 29 | 30 | ```sh 31 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=cypress/component 32 | ``` 33 | 34 | You can also use both `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` and `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` simultaneously. For example, to run all Cypress test files with `cy.js`, except those in the `cypress/e2e` directory, use: 35 | 36 | ```sh 37 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=**/*.cy.js 38 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=cypress/e2e 39 | ``` 40 | 41 | > [!TIP] 42 | > This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library. 43 | -------------------------------------------------------------------------------- /docs/gotest.md: -------------------------------------------------------------------------------- 1 | # Using bktec with go test 2 | 3 | To integrate `bktec` with Go's testing framework, you first need to install [`gotestsum`](https://github.com/gotestyourself/gotestsum), which `bktec` uses to generate JUnit XML reports. 4 | 5 | Set the following environment variables to configure `bktec` for your Go project: 6 | 7 | ```sh 8 | # Tell bktec to use the Go test runner integration 9 | export BUILDKITE_TEST_ENGINE_TEST_RUNNER=gotest 10 | 11 | # Specify where gotestsum should write the JUnit XML report 12 | # A unique file name per build is recommended, especially when running in parallel 13 | export BUILDKITE_TEST_ENGINE_RESULT_PATH=tmp/gotest-result.xml 14 | export BUILDKITE_TEST_ENGINE_SUITE_SLUG=your-suite-slug 15 | export BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN=your-token 16 | 17 | # Run the test engine client 18 | bktec 19 | ``` 20 | 21 | > [!IMPORTANT] 22 | > Due to Go's package-oriented design, file-level or example-level test splitting (like that available for RSpec or Pytest) is not supported. This means test splitting is less granular, and automatic retries operate on the entire package rather than individual tests. 23 | 24 | ## Configure test command 25 | 26 | By default, `bktec` runs go test with the following command: 27 | 28 | ```sh 29 | gotestsum --junitfile={{resultPath}} {{packages}} 30 | ``` 31 | 32 | In this command, `{{packages}}` is replaced by bktec with the list of packages to run, and `{{resultPath}}` is replaced with the `BUILDKITE_TEST_ENGINE_RESULT_PATH` environment variable. 33 | 34 | You can customize this command using the `BUILDKITE_TEST_ENGINE_TEST_CMD` environment variable. For example: 35 | ```sh 36 | export BUILDKITE_TEST_ENGINE_TEST_CMD="gotestsum --format="testname" --junitfile={{resultPath}} {{packages}}" 37 | ``` 38 | 39 | ## Filter packages 40 | 41 | Support for filtering specific packages is planned for a future release. Please let us know if this is a feature you need sooner. 42 | 43 | ## Test state management 44 | 45 | Using `bktec` allows you to manage test states, such as muting flaky tests, directly through the Buildkite Test Engine platform. This helps in managing test suites more effectively. 46 | 47 | ## Test splitting by package 48 | 49 | `bktec` supports package-level test splitting for Go tests. 50 | 51 | ```yaml 52 | - name: "Go test :golang:" 53 | commands: 54 | - bktec 55 | env: 56 | ... 57 | parallelism: 2 # This activate test splitting! 58 | ``` 59 | 60 | 61 | 62 | ## Automatically retry failed tests 63 | 64 | You can configure `bktec` to automatically retry failed tests using the `BUILDKITE_TEST_ENGINE_RETRY_COUNT` environment variable. 65 | When this variable is set to a number greater than `0`, `bktec` will retry each failed packages up to the specified number of times. 66 | 67 | To enable automatic retry, set the following environment variable: 68 | 69 | ```sh 70 | export BUILDKITE_TEST_ENGINE_RETRY_COUNT=1 71 | ``` 72 | 73 | ## Full Buildkite pipeline example 74 | 75 | ```yaml 76 | - name: "Go test :golang:" 77 | commands: 78 | - bktec 79 | env: 80 | BUILDKITE_ANALYTICS_TOKEN: your-suite-token # For test collector 81 | BUILDKITE_TEST_ENGINE_SUITE_SLUG: your-suite-slug 82 | BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN: your-api-token # For state management 83 | BUILDKITE_TEST_ENGINE_TEST_RUNNER: gotest 84 | BUILDKITE_TEST_ENGINE_RESULT_PATH: tmp/gotest-result.xml 85 | BUILDKITE_TEST_ENGINE_RETRY_COUNT: 1 86 | parallelism: 2 87 | plugins: 88 | # This will make sure test result are sent to buildkite. 89 | - test-collector#v1.11.0: 90 | files: "tmp/gotest-result.xml" 91 | format: "junit" 92 | ``` 93 | -------------------------------------------------------------------------------- /docs/jest.md: -------------------------------------------------------------------------------- 1 | # Using bktec with Jest 2 | To integrate bktec with Jest, set the `BUILDKITE_TEST_ENGINE_TEST_RUNNER` environment variable to `jest`. Then, specify the `BUILDKITE_TEST_ENGINE_RESULT_PATH` to define where the JSON result should be stored. bktec will instruct Jest to output the JSON result to this path, which is necessary for bktec to read the test results for retries and verification purposes. 3 | 4 | ```sh 5 | export BUILDKITE_TEST_ENGINE_TEST_RUNNER=jest 6 | export BUILDKITE_TEST_ENGINE_RESULT_PATH=tmp/jest-result.json 7 | ``` 8 | 9 | ## Configure test command 10 | By default, bktec runs Jest with the following command: 11 | 12 | ```sh 13 | npx jest {{testExamples}} --json --testLocationInResults --outputFile {{resultPath}} 14 | ``` 15 | 16 | In this command, `{{testExamples}}` is replaced by bktec with the list of test files or tests to run, and `{{resultPath}}` is replaced with the value set in `BUILDKITE_TEST_ENGINE_RESULT_PATH`. You can customize this command using the `BUILDKITE_TEST_ENGINE_TEST_CMD` environment variable. 17 | 18 | To customize the test command, set the following environment variable: 19 | ```sh 20 | export BUILDKITE_TEST_ENGINE_TEST_CMD="yarn test {{testExamples}} --json --testLocationInResults --outputFile {{resultPath}}" 21 | ``` 22 | 23 | > [!IMPORTANT] 24 | > Make sure to append `--json --testLocationInResults --outputFile {{resultPath}}` in your custom test command, as bktec requires this to read the test results for retries and verification purposes. 25 | 26 | ## Filter test files 27 | By default, bktec runs test files that match the `**/{__tests__/**/*,*.spec,*.test}.{ts,js,tsx,jsx}` pattern. You can customize this pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` environment variable. For instance, to configure bktec to only run Jest test files inside the `src/components` directory, use: 28 | 29 | ```sh 30 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=src/components/**/*.test.{ts,tsx} 31 | ``` 32 | 33 | Additionally, you can exclude specific files or directories that match a certain pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` environment variable. For example, to exclude test files inside the `src/utilities` directory, use: 34 | 35 | ```sh 36 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=src/utilities 37 | ``` 38 | 39 | You can also use both `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` and `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` simultaneously. For example, to run all Jest test files with `spec.ts`, except those in the `src/components` directory, use: 40 | 41 | ```sh 42 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=**/*.spec.ts 43 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=src/components 44 | ``` 45 | 46 | > [!TIP] 47 | > This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library. 48 | 49 | ## Automatically retry failed tests 50 | You can configure bktec to automatically retry failed tests using the `BUILDKITE_TEST_ENGINE_RETRY_COUNT` environment variable. When this variable is set to a number greater than `0`, bktec will retry each failed test up to the specified number of times, using the following command: 51 | 52 | ```sh 53 | npx yarn --testNamePattern '{{testNamePattern}}' --json --testLocationInResults --outputFile {{resultPath}} 54 | ``` 55 | 56 | In this command, `{{testNamePattern}}` is replaced by bktec with the list of failed tests to run, and `{{resultPath}}` is replaced with the value set in `BUILDKITE_TEST_ENGINE_RESULT_PATH`. You can customize this command using the `BUILDKITE_TEST_ENGINE_RETRY_CMD` environment variable. 57 | 58 | To enable automatic retry and customize the retry command, set the following environment variable: 59 | ```sh 60 | export BUILDKITE_TEST_ENGINE_RETRY_CMD="yarn test --testNamePattern '{{testNamePattern}}' --json --testLocationInResults --outputFile {{resultPath}}" 61 | export BUILDKITE_TEST_ENGINE_RETRY_COUNT=2 62 | ``` 63 | 64 | To limit the number of files that Jest parses when retrying tests, you can also include a `{{testExamples}}` placeholder to specify the filenames of retried tests: 65 | ```sh 66 | export BUILDKITE_TEST_ENGINE_RETRY_CMD="yarn test {{testExamples}} --testNamePattern '{{testNamePattern}}' --json --testLocationInResults --outputFile {{resultPath}}" 67 | export BUILDKITE_TEST_ENGINE_RETRY_COUNT=2 68 | ``` 69 | 70 | > [!IMPORTANT] 71 | > Make sure to append `--testNamePattern '{{testNamePattern}}' --json --testLocationInResults --outputFile {{resultPath}}` in your custom retry command. 72 | -------------------------------------------------------------------------------- /docs/playwright.md: -------------------------------------------------------------------------------- 1 | # Using bktec with Playwright 2 | To integrate bktec with Playwright, start by configuring Playwright to output the results to a JSON file. This is necessary for bktec to read the test results for retries and verification purposes. 3 | 4 | ```js 5 | // playwright.config.js 6 | import { defineConfig } from '@playwright/test'; 7 | 8 | export default defineConfig({ 9 | reporter: [ 10 | ['json', { outputFile: './tmp/test-results.json' }] 11 | ], 12 | }); 13 | ``` 14 | 15 | Next, set the `BUILDKITE_TEST_ENGINE_RESULT_PATH` environment variable to the path of your JSON file. 16 | 17 | ```sh 18 | export BUILDKITE_TEST_ENGINE_TEST_RUNNER=playwright 19 | export BUILDKITE_TEST_ENGINE_RESULT_PATH=./tmp/test-results.json 20 | ``` 21 | 22 | ## Configure test command 23 | By default, bktec runs Playwright with the following command: 24 | 25 | ```sh 26 | npx playwright test {{testExamples}} 27 | ``` 28 | 29 | In this command, `{{testExamples}}` is replaced by bktec with the list of test files to run. You can customize this command using the `BUILDKITE_TEST_ENGINE_TEST_CMD` environment variable. 30 | 31 | To customize the test command, set the following environment variable: 32 | ```sh 33 | export BUILDKITE_TEST_ENGINE_TEST_CMD="yarn test {{testExamples}}" 34 | ``` 35 | 36 | ## Filter test files 37 | By default, bktec runs test files that match the `**/{*.spec,*.test}.{ts,js}` pattern. You can customize this pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` environment variable. For instance, to configure bktec to only run Playwright test files inside the `tests` directory, use: 38 | 39 | ```sh 40 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=tests/**/*.test.ts 41 | ``` 42 | 43 | Additionally, you can exclude specific files or directories that match a certain pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` environment variable. For example, to exclude test files inside the `src/components` directory, use: 44 | 45 | ```sh 46 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=src/components 47 | ``` 48 | 49 | You can also use both `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` and `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` simultaneously. For example, to run all Playwright test files with `.spec.ts` extension, except those in the `src/components` directory, use: 50 | 51 | ```sh 52 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=**/*.spec.ts 53 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=src/components 54 | ``` 55 | 56 | > [!TIP] 57 | > This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library. 58 | 59 | ## Automatically retry failed tests 60 | You can configure bktec to automatically retry failed tests using the `BUILDKITE_TEST_ENGINE_RETRY_COUNT` environment variable. When this variable is set to a number greater than `0`, bktec will retry each failed test up to the specified number of times, using either the default test command or the command specified in `BUILDKITE_TEST_ENGINE_TEST_CMD`. 61 | 62 | To enable automatic retry, set the following environment variable: 63 | ```sh 64 | export BUILDKITE_TEST_ENGINE_RETRY_COUNT=2 65 | ``` 66 | -------------------------------------------------------------------------------- /docs/pytest-pants.md: -------------------------------------------------------------------------------- 1 | # Using bktec with pants (Experimental) 2 | 3 | > [!WARNING] 4 | > Pants support is currently experimental and has limited feature support. Only the following features are supported: 5 | > 6 | > - Automatically retry failed tests 7 | > - Mute tests (ignore test failures) 8 | > 9 | > The following features are not supported: 10 | > 11 | > - Filter test files 12 | > - Split slow files by individual test example 13 | > - Skip tests 14 | 15 | To integrate bktec with pants, you need to [install and configure Buildkite Test Collector for pytest](https://buildkite.com/docs/test-engine/python-collectors#pytest-collector) first. Then set the `BUILDKITE_TEST_ENGINE_TEST_RUNNER` environment variable to `pytest-pants`. 16 | 17 | Look at the example configuration files in the [pytest_pants testdata directory](../internal/runner/testdata/pytest_pants) for an example of how to add buildkite-test-collector to the pants resolve used by pytest. Specifically: 18 | 19 | - [pants.toml](../internal/runner/testdata/pytest_pants/pants.toml) - pants configuration 20 | - [3rdparty/python/BUILD](../internal/runner/testdata/pytest_pants/3rdparty/python/BUILD) - python_requirement targets 21 | - [3rdparty/python/pytest-requirements.txt](../internal/runner/testdata/pytest_pants/3rdparty/python/pytest-requirements.txt) - Python requirements.txt 22 | 23 | In the example in the repository, you would need to generate a lockfile next, i.e. 24 | 25 | ```sh 26 | pants generate-lockfiles --resolve=pytest 27 | ``` 28 | 29 | Only running `pants test` with `python_test` targets is supported at this time. 30 | 31 | ```sh 32 | export BUILDKITE_TEST_ENGINE_TEST_RUNNER=pytest-pants 33 | export BUILDKITE_TEST_ENGINE_TEST_CMD="pants --filter-target-type=python_test --changed-since=HEAD~1 test -- --json={{resultPath}} --merge-json" 34 | bktec 35 | ``` 36 | 37 | ## Configure test command 38 | 39 | While pants support is experimental there is no default command. That means it is required to set `BUILDKITE_TEST_ENGINE_TEST_CMD`. 40 | Below are a few recommendations for specific scenarios: 41 | 42 | --- 43 | 44 | ```sh 45 | export BUILDKITE_TEST_ENGINE_TEST_CMD="pants --filter-target-type=python_test test //:: -- --json={{resultPath}} --merge-json"" 46 | ``` 47 | 48 | This command is a good option if you want to run all python tests in your repository. 49 | 50 | --- 51 | 52 | ```sh 53 | export BUILDKITE_TEST_ENGINE_TEST_CMD="pants --filter-target-type=python_test --changed-since=HEAD~1 test -- --json={{resultPath}} --merge-json" 54 | ``` 55 | 56 | This command is a good option if you want to only run the python tests that were 57 | impacted by any changes made since `HEAD~1`. Checkout [pants Advanced target 58 | selection doc][pants-advanced-target-selection] for more information on 59 | `--changed-since`. 60 | 61 | --- 62 | 63 | In both commands, `{{resultPath}}` is replaced with a unique temporary path created by bktec. `--json` option is a custom pytest option added by Buildkite Test Collector to save the result into a JSON file at given path. You can further customize the test command for your specific use case. 64 | 65 | > [!IMPORTANT] 66 | > Make sure to append `-- --json={{resultPath}} --merge-json` in your custom pants test command, as bktec requires these options to read the test results for retries and verification purposes. 67 | 68 | ## Filter test files 69 | 70 | There is not support for filtering test files at this time. 71 | 72 | ## Automatically retry failed tests 73 | 74 | You can configure bktec to automatically retry failed tests using the `BUILDKITE_TEST_ENGINE_RETRY_COUNT` environment variable. When this variable is set to a number greater than `0`, bktec will retry each failed test up to the specified number of times, using either the default test command or the command specified in `BUILDKITE_TEST_ENGINE_TEST_CMD`. Because pants caches test results, only failed tests will be retried. 75 | 76 | To enable automatic retry, set the following environment variable: 77 | 78 | ```sh 79 | export BUILDKITE_TEST_ENGINE_RETRY_COUNT=2 80 | ``` 81 | 82 | [pants-advanced-target-selection]: https://www.pantsbuild.org/stable/docs/using-pants/advanced-target-selection 83 | -------------------------------------------------------------------------------- /docs/pytest.md: -------------------------------------------------------------------------------- 1 | # Using bktec with pytest 2 | To integrate bktec with pytest, you need to [install and configure Buildkite Test Collector for pytest](https://buildkite.com/docs/test-engine/python-collectors#pytest-collector) first. Then set the `BUILDKITE_TEST_ENGINE_TEST_RUNNER` environment variable to `pytest`. 3 | 4 | ```sh 5 | export BUILDKITE_TEST_ENGINE_TEST_RUNNER=pytest 6 | bktec 7 | ``` 8 | 9 | ## Configure test command 10 | By default, bktec runs pytest with the following command: 11 | 12 | ```sh 13 | pytest {{testExamples}} --json={{resultPath}} 14 | ``` 15 | 16 | In this command, `{{testExamples}}` is replaced by bktec with the list of test files or tests to run, and `{{resultPath}}` is replaced with a unique temporary path created by bktec. `--json` option is a custom option added by Buildkite Test Collector to save the result into a JSON file at given path. You can customize this command using the `BUILDKITE_TEST_ENGINE_TEST_CMD` environment variable. 17 | 18 | To customize the test command, set the following environment variable: 19 | ```sh 20 | export BUILDKITE_TEST_ENGINE_TEST_CMD="pytest --cache-clear --json={{resultPath}} {{testExamples}}" 21 | ``` 22 | 23 | > [!IMPORTANT] 24 | > Make sure to include `--json={{resultPath}}` in your custom test command, as bktec requires this to read the test results for retries and verification purposes. 25 | 26 | ## Filter test files 27 | By default, bktec runs test files that match the `**/{*_test,test_*}.py` pattern. You can customize this pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` environment variable. For instance, to configure bktec to only run test files inside the `tests` directory, use: 28 | 29 | ```sh 30 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN="tests/**/{*_test,test_*}.py" 31 | ``` 32 | 33 | Additionally, you can exclude specific files or directories that match a certain pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` environment variable. For example, to exclude test files inside the `tests/api` directory, use: 34 | 35 | ```sh 36 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=tests/api 37 | ``` 38 | 39 | You can also use both `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` and `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` simultaneously. For example, to run all test files inside the `tests/` directory, except those inside `tests/api`, use: 40 | 41 | ```sh 42 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN="**/{*_test,test_*}.py" 43 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=tests/api 44 | ``` 45 | 46 | > [!TIP] 47 | > This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library. 48 | 49 | ## Automatically retry failed tests 50 | You can configure bktec to automatically retry failed tests using the `BUILDKITE_TEST_ENGINE_RETRY_COUNT` environment variable. When this variable is set to a number greater than `0`, bktec will retry each failed test up to the specified number of times, using either the default test command or the command specified in `BUILDKITE_TEST_ENGINE_TEST_CMD`. 51 | 52 | To enable automatic retry, set the following environment variable: 53 | ```sh 54 | export BUILDKITE_TEST_ENGINE_RETRY_COUNT=2 55 | ``` 56 | -------------------------------------------------------------------------------- /docs/rspec.md: -------------------------------------------------------------------------------- 1 | # Using bktec with RSpec 2 | To integrate bktec with RSpec, set the `BUILDKITE_TEST_ENGINE_TEST_RUNNER` environment variable to `rspec`. Then, specify the `BUILDKITE_TEST_ENGINE_RESULT_PATH` to define where the JSON result should be stored. bktec will instruct RSpec to output the JSON result to this path, which is necessary for bktec to read the test results for retries and verification purposes. 3 | 4 | ```sh 5 | export BUILDKITE_TEST_ENGINE_TEST_RUNNER=rspec 6 | export BUILDKITE_TEST_ENGINE_RESULT_PATH=tmp/result.json 7 | ``` 8 | 9 | ## Configure test command 10 | By default, bktec runs RSpec with the following command: 11 | 12 | ```sh 13 | bundle exec rspec --format progress --format json --out {{resultPath}} {{testExamples}} 14 | ``` 15 | 16 | In this command, `{{testExamples}}` is replaced by bktec with the list of test files or tests to run, and `{{resultPath}}` is replaced with the value set in `BUILDKITE_TEST_ENGINE_RESULT_PATH`. You can customize this command using the `BUILDKITE_TEST_ENGINE_TEST_CMD` environment variable. 17 | 18 | To customize the test command, set the following environment variable: 19 | ```sh 20 | export BUILDKITE_TEST_ENGINE_TEST_CMD="bin/rspec --format json --out {{resultPath}} {{testExamples}}" 21 | ``` 22 | 23 | > [!IMPORTANT] 24 | > Make sure to append `--format json --out {{resultPath}}` in your custom test command, as bktec requires this to read the test results for retries and verification purposes. 25 | 26 | > [!IMPORTANT] 27 | > If you have another formatter configured in an [RSpec configuration file](https://rspec.info/features/3-13/rspec-core/configuration/read-options-from-file/), the default test command will override it. To avoid this, use a custom test command and add a JSON formatter in your RSpec configuration file. 28 | 29 | ```sh 30 | export BUILDKITE_TEST_ENGINE_RESULT_PATH=tmp/rspec-result.json 31 | export BUILDKITE_TEST_ENGINE_TEST_CMD="bundle exec rspec {{testExamples}}" 32 | ``` 33 | 34 | Then, in your RSpec configuration file: 35 | 36 | ```sh 37 | #.rspec 38 | --format junit 39 | --out rspec.xml 40 | --format json 41 | --out tmp/rspec-result.json 42 | ``` 43 | 44 | ## Filter test files 45 | By default, bktec runs test files that match the `spec/**/*_spec.rb` pattern. You can customize this pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` environment variable. For instance, to configure bktec to only run test files inside the `spec/features` directory, use: 46 | 47 | ```sh 48 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=spec/features/**/*_spec.rb 49 | ``` 50 | 51 | Additionally, you can exclude specific files or directories that match a certain pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` environment variable. For example, to exclude test files inside the `spec/features` directory, use: 52 | 53 | ```sh 54 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=spec/features 55 | ``` 56 | 57 | You can also use both `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` and `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN` simultaneously. For example, to run all test files inside the `spec/models` directory, except those inside `spec/models/user`, use: 58 | 59 | ```sh 60 | export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=spec/models/**/*_spec.rb 61 | export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=spec/models/user 62 | ``` 63 | 64 | > [!TIP] 65 | > This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library. 66 | 67 | ## Automatically retry failed tests 68 | You can configure bktec to automatically retry failed tests using the `BUILDKITE_TEST_ENGINE_RETRY_COUNT` environment variable. When this variable is set to a number greater than `0`, bktec will retry each failed test up to the specified number of times, using the command set in `BUILDKITE_TEST_ENGINE_RETRY_CMD` environment variable. If this variable is not set, bktec will use either the default test command or the command specified in `BUILDKITE_TEST_ENGINE_TEST_CMD` to retry the tests. 69 | 70 | To enable automatic retry, set the following environment variable: 71 | ```sh 72 | export BUILDKITE_TEST_ENGINE_RETRY_COUNT=2 73 | ``` 74 | 75 | ## Split slow files by individual test example 76 | By default, bktec splits your test suite into batches of test files. In some scenarios, e.g. if your test suite has a few test files that take a very long time to run, you may want to split slow test files into individual test examples for execution. To enable this, you can set the `BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE` environment variable to `true`. This setting enables bktec to dynamically split slow test files across multiple partitions based on their duration and the number of parallelism. 77 | 78 | To enable split by example, set the following environment variable: 79 | ```sh 80 | export BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE=true 81 | ``` 82 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/buildkite/test-engine-client 2 | 3 | go 1.24 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | github.com/buildkite/roko v1.3.1 9 | github.com/google/go-cmp v0.7.0 10 | ) 11 | 12 | require ( 13 | drjosh.dev/zzglob v0.4.0 14 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 15 | github.com/olekukonko/tablewriter v0.0.5 16 | github.com/pact-foundation/pact-go/v2 v2.4.1 17 | github.com/stretchr/testify v1.10.0 18 | ) 19 | 20 | require ( 21 | github.com/davecgh/go-spew v1.1.1 // indirect 22 | github.com/hashicorp/go-version v1.7.0 // indirect 23 | github.com/hashicorp/logutils v1.0.0 // indirect 24 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 25 | github.com/kr/text v0.2.0 // indirect 26 | github.com/mattn/go-runewidth v0.0.9 // indirect 27 | github.com/pmezard/go-difflib v1.0.0 // indirect 28 | github.com/spf13/afero v1.12.0 // indirect 29 | github.com/spf13/cobra v1.9.1 // indirect 30 | github.com/spf13/pflag v1.0.6 // indirect 31 | golang.org/x/net v0.38.0 // indirect 32 | golang.org/x/sys v0.33.0 // indirect 33 | golang.org/x/text v0.23.0 // indirect 34 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect 35 | google.golang.org/grpc v1.71.0 // indirect 36 | google.golang.org/protobuf v1.36.5 // indirect 37 | gopkg.in/yaml.v3 v3.0.1 // indirect 38 | ) 39 | -------------------------------------------------------------------------------- /internal/api/client.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | "regexp" 12 | "runtime" 13 | "strconv" 14 | "time" 15 | 16 | "github.com/buildkite/roko" 17 | "github.com/buildkite/test-engine-client/internal/debug" 18 | "github.com/buildkite/test-engine-client/internal/version" 19 | ) 20 | 21 | // client is a client for the test plan API. 22 | // It contains the organization slug, server base URL, and an HTTP client. 23 | type Client struct { 24 | OrganizationSlug string 25 | ServerBaseUrl string 26 | httpClient *http.Client 27 | } 28 | 29 | // ClientConfig is the configuration for the test plan API client. 30 | type ClientConfig struct { 31 | AccessToken string 32 | OrganizationSlug string 33 | ServerBaseUrl string 34 | } 35 | 36 | // authTransport is a middleware for the HTTP client. 37 | type authTransport struct { 38 | accessToken string 39 | } 40 | 41 | // RoundTrip adds the Authorization header to all requests made by the HTTP client. 42 | func (t *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { 43 | req.Header.Set("Authorization", "Bearer "+t.accessToken) 44 | req.Header.Set("User-Agent", fmt.Sprintf( 45 | "Buildkite Test Engine Client/%s (%s/%s)", 46 | version.Version, runtime.GOOS, runtime.GOARCH, 47 | )) 48 | return http.DefaultTransport.RoundTrip(req) 49 | } 50 | 51 | // NewClient creates a new client for the test plan API with the given configuration. 52 | // It also creates an HTTP client with an authTransport middleware. 53 | func NewClient(cfg ClientConfig) *Client { 54 | httpClient := &http.Client{ 55 | Transport: &authTransport{ 56 | accessToken: cfg.AccessToken, 57 | }, 58 | } 59 | 60 | return &Client{ 61 | OrganizationSlug: cfg.OrganizationSlug, 62 | ServerBaseUrl: cfg.ServerBaseUrl, 63 | httpClient: httpClient, 64 | } 65 | } 66 | 67 | var ( 68 | retryTimeout = 130 * time.Second 69 | initialDelay = 3000 * time.Millisecond 70 | ) 71 | 72 | var ErrRetryTimeout = errors.New("request retry timeout") 73 | 74 | type BillingError struct { 75 | Message string 76 | } 77 | 78 | func (e *BillingError) Error() string { 79 | return e.Message 80 | } 81 | 82 | type responseError struct { 83 | Message string `json:"message"` 84 | } 85 | 86 | func (e *responseError) Error() string { 87 | return e.Message 88 | } 89 | 90 | type httpRequest struct { 91 | Method string 92 | URL string 93 | Body any 94 | } 95 | 96 | // DoWithRetry sends http request with retries. 97 | // Successful API response (status code 200) is JSON decoded and stored in the value pointed to by v. 98 | // The request will be retried when the server returns 429 or 5xx status code, or when there is a network error. 99 | // After reaching the retry timeout, the function will return ErrRetryTimeout. 100 | // The request will not be retried when the server returns 4xx status code, 101 | // and the error message will be returned as an error. 102 | func (c *Client) DoWithRetry(ctx context.Context, reqOptions httpRequest, v interface{}) (*http.Response, error) { 103 | r := roko.NewRetrier( 104 | roko.TryForever(), 105 | roko.WithStrategy(roko.ExponentialSubsecond(initialDelay)), 106 | roko.WithJitter(), 107 | ) 108 | 109 | retryContext, cancelRetryContext := context.WithTimeout(ctx, retryTimeout) 110 | defer cancelRetryContext() 111 | 112 | // retry loop 113 | debug.Printf("Sending request %s %s", reqOptions.Method, reqOptions.URL) 114 | resp, err := roko.DoFunc(retryContext, r, func(r *roko.Retrier) (*http.Response, error) { 115 | if r.AttemptCount() > 0 { 116 | debug.Printf("Retrying requests, attempt %d", r.AttemptCount()) 117 | } 118 | 119 | // Each request times out after 15 seconds, chosen to provide some 120 | // headroom on top of the goal p99 time to fetch of 10s. 121 | reqContext, cancelReqContext := context.WithTimeout(ctx, 15*time.Second) 122 | defer cancelReqContext() 123 | 124 | req, err := http.NewRequestWithContext(reqContext, reqOptions.Method, reqOptions.URL, nil) 125 | if err != nil { 126 | r.Break() 127 | return nil, fmt.Errorf("creating request: %w", err) 128 | } 129 | 130 | if reqOptions.Method != http.MethodGet && reqOptions.Body != nil { 131 | // add body to request 132 | reqBody, err := json.Marshal(reqOptions.Body) 133 | if err != nil { 134 | r.Break() 135 | return nil, fmt.Errorf("converting body to json: %w", err) 136 | } 137 | req.Body = io.NopCloser(bytes.NewReader(reqBody)) 138 | } 139 | 140 | req.Header.Add("Content-Type", "application/json") 141 | 142 | resp, err := c.httpClient.Do(req) 143 | 144 | // If we get an error before getting a response, 145 | // which means there is a network error (e.g. protocol error, timeout), 146 | // we should return and retry. 147 | if err != nil { 148 | debug.Printf("Error sending request: %v", err) 149 | return nil, err 150 | } 151 | 152 | debug.Printf("Response code %d", resp.StatusCode) 153 | 154 | // If we get a 429, we should return and retry after the rate limit resets. 155 | if resp.StatusCode == http.StatusTooManyRequests { 156 | if rateLimitReset, err := strconv.Atoi(resp.Header.Get("RateLimit-Reset")); err == nil { 157 | r.SetNextInterval(time.Duration(rateLimitReset) * time.Second) 158 | } 159 | return resp, fmt.Errorf("response code: 429") 160 | } 161 | 162 | // If we get a 409, we aren't the first client to create the plan so return 163 | // and retry 164 | if resp.StatusCode == http.StatusConflict { 165 | return resp, fmt.Errorf("response code: %d", resp.StatusCode) 166 | } 167 | 168 | // If we get a 5xx, we should return and retry 169 | if resp.StatusCode >= 500 { 170 | return resp, fmt.Errorf("response code: %d", resp.StatusCode) 171 | } 172 | 173 | // Other than above cases, we should break from the retry loop. 174 | r.Break() 175 | 176 | responseBody, err := io.ReadAll(resp.Body) 177 | if err != nil { 178 | return nil, fmt.Errorf("reading response body: %w", err) 179 | } 180 | defer resp.Body.Close() 181 | 182 | if resp.StatusCode != http.StatusOK { 183 | var respError responseError 184 | err = json.Unmarshal(responseBody, &respError) 185 | if err != nil { 186 | return resp, fmt.Errorf("parsing response: %w", err) 187 | } 188 | 189 | if matched := regexp.MustCompile(`^Billing Error`).MatchString(respError.Message); matched && resp.StatusCode == 403 { 190 | return resp, &BillingError{Message: respError.Message} 191 | } 192 | 193 | return resp, &respError 194 | } 195 | 196 | // parse response 197 | if v != nil && len(responseBody) > 0 { 198 | err = json.Unmarshal(responseBody, v) 199 | if err != nil { 200 | return nil, fmt.Errorf("parsing response: %w", err) 201 | } 202 | } 203 | 204 | return resp, nil 205 | }) 206 | 207 | if errors.Is(err, context.DeadlineExceeded) { 208 | return resp, ErrRetryTimeout 209 | } 210 | 211 | return resp, err 212 | } 213 | -------------------------------------------------------------------------------- /internal/api/create_test_plan.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/buildkite/test-engine-client/internal/plan" 9 | ) 10 | 11 | type TestPlanParamsTest struct { 12 | Files []plan.TestCase `json:"files"` 13 | Examples []plan.TestCase `json:"examples,omitempty"` 14 | } 15 | 16 | // TestPlanParams represents the config params sent when fetching a test plan. 17 | type TestPlanParams struct { 18 | Runner string `json:"runner"` 19 | Identifier string `json:"identifier"` 20 | Parallelism int `json:"parallelism"` 21 | Branch string `json:"branch"` 22 | Tests TestPlanParamsTest `json:"tests"` 23 | } 24 | 25 | // CreateTestPlan creates a test plan from the server. 26 | // ErrRetryTimeout is returned if the client failed to communicate with the server after exceeding the retry limit. 27 | func (c Client) CreateTestPlan(ctx context.Context, suiteSlug string, params TestPlanParams) (plan.TestPlan, error) { 28 | postUrl := fmt.Sprintf("%s/v2/analytics/organizations/%s/suites/%s/test_plan", c.ServerBaseUrl, c.OrganizationSlug, suiteSlug) 29 | 30 | var testPlan plan.TestPlan 31 | _, err := c.DoWithRetry(ctx, httpRequest{ 32 | Method: http.MethodPost, 33 | URL: postUrl, 34 | Body: params, 35 | }, &testPlan) 36 | 37 | if err != nil { 38 | return plan.TestPlan{}, err 39 | } 40 | 41 | return testPlan, nil 42 | } 43 | -------------------------------------------------------------------------------- /internal/api/doc.go: -------------------------------------------------------------------------------- 1 | // Package api provides an API client for the Test Plan API. 2 | package api 3 | -------------------------------------------------------------------------------- /internal/api/fetch_files_timing.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | ) 9 | 10 | type fetchFilesTimingParams struct { 11 | Paths []string `json:"paths"` 12 | } 13 | 14 | // FetchFilesTiming fetches the timing of the requested files from the server. 15 | // The server only returns timings for the files that has been run before. 16 | // ErrRetryTimeout is returned if the client failed to communicate with the server after exceeding the retry limit. 17 | func (c Client) FetchFilesTiming(ctx context.Context, suiteSlug string, files []string) (map[string]time.Duration, error) { 18 | url := fmt.Sprintf("%s/v2/analytics/organizations/%s/suites/%s/test_files", c.ServerBaseUrl, c.OrganizationSlug, suiteSlug) 19 | 20 | var filesTiming map[string]int 21 | _, err := c.DoWithRetry(ctx, httpRequest{ 22 | Method: http.MethodPost, 23 | URL: url, 24 | Body: fetchFilesTimingParams{ 25 | Paths: files, 26 | }, 27 | }, &filesTiming) 28 | 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | result := map[string]time.Duration{} 34 | for path, duration := range filesTiming { 35 | result[path] = time.Duration(duration * int(time.Millisecond)) 36 | } 37 | 38 | return result, nil 39 | } 40 | -------------------------------------------------------------------------------- /internal/api/fetch_files_timing_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | "time" 11 | 12 | "github.com/google/go-cmp/cmp" 13 | "github.com/pact-foundation/pact-go/v2/consumer" 14 | "github.com/pact-foundation/pact-go/v2/matchers" 15 | ) 16 | 17 | func TestFetchFilesTiming(t *testing.T) { 18 | mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{ 19 | Consumer: "TestEngineClient", 20 | Provider: "TestEngineServer", 21 | }) 22 | 23 | if err != nil { 24 | t.Error("Error mocking provider", err) 25 | } 26 | 27 | files := []string{"apple_spec.rb", "banana_spec.rb", "cherry_spec.rb", "dragonfruit_spec.rb"} 28 | 29 | err = mockProvider. 30 | AddInteraction(). 31 | Given("Test file timings exist"). 32 | UponReceiving("A request for test file timings"). 33 | WithRequest("POST", "/v2/analytics/organizations/buildkite/suites/rspec/test_files", func(b *consumer.V2RequestBuilder) { 34 | b.Header("Authorization", matchers.Like("Bearer asdf1234")) 35 | b.JSONBody(fetchFilesTimingParams{ 36 | Paths: files, 37 | }) 38 | }). 39 | WillRespondWith(200, func(b *consumer.V2ResponseBuilder) { 40 | b.Header("Content-Type", matchers.Like("application/json; charset=utf-8")) 41 | b.JSONBody(matchers.MapMatcher{ 42 | "./apple_spec.rb": matchers.Like(1121), 43 | "./banana_spec.rb": matchers.Like(3121), 44 | "./cherry_spec.rb": matchers.Like(2143), 45 | }) 46 | }). 47 | ExecuteTest(t, func(config consumer.MockServerConfig) error { 48 | url := fmt.Sprintf("http://%s:%d", config.Host, config.Port) 49 | c := NewClient(ClientConfig{ 50 | AccessToken: "asdf1234", 51 | OrganizationSlug: "buildkite", 52 | ServerBaseUrl: url, 53 | }) 54 | got, err := c.FetchFilesTiming(context.Background(), "rspec", files) 55 | if err != nil { 56 | t.Errorf("FetchFilesTiming() error = %v", err) 57 | } 58 | want := map[string]time.Duration{ 59 | "./apple_spec.rb": 1121 * time.Millisecond, 60 | "./banana_spec.rb": 3121 * time.Millisecond, 61 | "./cherry_spec.rb": 2143 * time.Millisecond, 62 | } 63 | if diff := cmp.Diff(got, want); diff != "" { 64 | t.Errorf("FetchFilesTiming() diff (-got +want):\n%s", diff) 65 | } 66 | return nil 67 | }) 68 | 69 | if err != nil { 70 | t.Error(err) 71 | } 72 | } 73 | 74 | func TestFetchFilesTiming_BadRequest(t *testing.T) { 75 | requestCount := 0 76 | svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 77 | requestCount++ 78 | http.Error(w, `{"message": "bad request"}`, http.StatusBadRequest) 79 | })) 80 | defer svr.Close() 81 | 82 | c := NewClient(ClientConfig{ 83 | OrganizationSlug: "my-org", 84 | ServerBaseUrl: svr.URL, 85 | }) 86 | 87 | files := []string{"apple_spec.rb", "banana_spec.rb"} 88 | _, err := c.FetchFilesTiming(context.Background(), "my-suite", files) 89 | 90 | if requestCount > 1 { 91 | t.Errorf("http request count = %v, want %d", requestCount, 1) 92 | } 93 | 94 | if err.Error() != "bad request" { 95 | t.Errorf("FetchFilesTiming() error = %v, want %v", err, ErrRetryTimeout) 96 | } 97 | } 98 | 99 | func TestFetchFilesTiming_InternalServerError(t *testing.T) { 100 | originalTimeout := retryTimeout 101 | retryTimeout = 1 * time.Millisecond 102 | t.Cleanup(func() { 103 | retryTimeout = originalTimeout 104 | }) 105 | 106 | svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 107 | http.Error(w, `{"message": "something went wrong"}`, http.StatusInternalServerError) 108 | })) 109 | defer svr.Close() 110 | 111 | c := NewClient(ClientConfig{ 112 | OrganizationSlug: "my-org", 113 | ServerBaseUrl: svr.URL, 114 | }) 115 | 116 | files := []string{"apple_spec.rb", "banana_spec.rb"} 117 | _, err := c.FetchFilesTiming(context.Background(), "my-suite", files) 118 | 119 | if !errors.Is(err, ErrRetryTimeout) { 120 | t.Errorf("FetchFilesTiming() error = %v, want %v", err, ErrRetryTimeout) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /internal/api/fetch_test_plan.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/buildkite/test-engine-client/internal/plan" 9 | ) 10 | 11 | // FetchTestPlan fetchs a test plan from the server. 12 | // ErrRetryTimeout is returned if the client failed to communicate with the server after exceeding the retry limit. 13 | func (c Client) FetchTestPlan(ctx context.Context, suiteSlug string, identifier string, jobRetryCount int) (*plan.TestPlan, error) { 14 | url := fmt.Sprintf("%s/v2/analytics/organizations/%s/suites/%s/test_plan?identifier=%s&job_retry_count=%d", c.ServerBaseUrl, c.OrganizationSlug, suiteSlug, identifier, jobRetryCount) 15 | 16 | var testPlan plan.TestPlan 17 | 18 | resp, err := c.DoWithRetry(ctx, httpRequest{ 19 | Method: http.MethodGet, 20 | URL: url, 21 | }, &testPlan) 22 | 23 | if err != nil { 24 | if resp != nil && resp.StatusCode == http.StatusNotFound { 25 | return nil, nil 26 | } 27 | return nil, err 28 | } 29 | 30 | return &testPlan, nil 31 | } 32 | -------------------------------------------------------------------------------- /internal/api/fetch_test_plan_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | "time" 11 | 12 | "github.com/buildkite/test-engine-client/internal/plan" 13 | "github.com/google/go-cmp/cmp" 14 | "github.com/pact-foundation/pact-go/v2/consumer" 15 | "github.com/pact-foundation/pact-go/v2/matchers" 16 | ) 17 | 18 | func TestFetchTestPlan(t *testing.T) { 19 | mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{ 20 | Consumer: "TestEngineClient", 21 | Provider: "TestEngineServer", 22 | }) 23 | 24 | if err != nil { 25 | t.Error("Error mocking provider", err) 26 | } 27 | 28 | err = mockProvider. 29 | AddInteraction(). 30 | Given("A test plan exists"). 31 | UponReceiving("A request for test plan with identifier abc123"). 32 | WithRequest("GET", "/v2/analytics/organizations/buildkite/suites/rspec/test_plan", func(b *consumer.V2RequestBuilder) { 33 | b.Header("Authorization", matchers.Like("Bearer asdf1234")) 34 | b.Query("identifier", matchers.Like("abc123")) 35 | b.Query("job_retry_count", matchers.Like("0")) 36 | }). 37 | WillRespondWith(200, func(b *consumer.V2ResponseBuilder) { 38 | b.Header("Content-Type", matchers.Like("application/json; charset=utf-8")) 39 | b.JSONBody(matchers.MapMatcher{ 40 | "tasks": matchers.Like(map[string]interface{}{ 41 | "1": matchers.Like(map[string]interface{}{ 42 | "node_number": matchers.Like(1), 43 | "tests": matchers.EachLike(matchers.MapMatcher{ 44 | "path": matchers.Like("sky_spec.rb:2"), 45 | "format": matchers.Like("example"), 46 | "estimated_duration": matchers.Like(1000), 47 | "identifier": matchers.Like("sky_spec.rb[1,1]"), 48 | "name": matchers.Like("is blue"), 49 | "scope": matchers.Like("sky"), 50 | }, 1), 51 | }), 52 | }), 53 | }) 54 | }). 55 | ExecuteTest(t, func(config consumer.MockServerConfig) error { 56 | url := fmt.Sprintf("http://%s:%d", config.Host, config.Port) 57 | 58 | cfg := ClientConfig{ 59 | AccessToken: "asdf1234", 60 | OrganizationSlug: "buildkite", 61 | ServerBaseUrl: url, 62 | } 63 | 64 | c := NewClient(cfg) 65 | 66 | got, err := c.FetchTestPlan(context.Background(), "rspec", "abc123", 0) 67 | 68 | if err != nil { 69 | t.Errorf("FetchTestPlan() error = %v", err) 70 | } 71 | 72 | want := plan.TestPlan{ 73 | Tasks: map[string]*plan.Task{ 74 | "1": { 75 | NodeNumber: 1, 76 | Tests: []plan.TestCase{{ 77 | Path: "sky_spec.rb:2", 78 | Identifier: "sky_spec.rb[1,1]", 79 | Name: "is blue", 80 | Scope: "sky", 81 | Format: "example", 82 | EstimatedDuration: 1000, 83 | }}, 84 | }, 85 | }, 86 | } 87 | 88 | if diff := cmp.Diff(got, &want); diff != "" { 89 | t.Errorf("FetchTestPlan() diff (-got +want):\n%s", diff) 90 | } 91 | 92 | return nil 93 | }) 94 | 95 | if err != nil { 96 | t.Error("mockProvider error", err) 97 | } 98 | } 99 | 100 | func TestFetchTestPlan_NotFound(t *testing.T) { 101 | mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{ 102 | Consumer: "TestEngineClient", 103 | Provider: "TestEngineServer", 104 | }) 105 | 106 | if err != nil { 107 | t.Error("Error mocking provider", err) 108 | } 109 | 110 | err = mockProvider. 111 | AddInteraction(). 112 | Given("A test plan doesn't exist"). 113 | UponReceiving("A request for test plan with identifier abc123"). 114 | WithRequest("GET", "/v2/analytics/organizations/buildkite/suites/rspec/test_plan", func(b *consumer.V2RequestBuilder) { 115 | b. 116 | Header("Authorization", matchers.Like("Bearer asdf1234")). 117 | Query("identifier", matchers.Like("abc123")). 118 | Query("job_retry_count", matchers.Like("0")) 119 | }). 120 | WillRespondWith(404, func(b *consumer.V2ResponseBuilder) { 121 | b.Header("Content-Type", matchers.Like("application/json; charset=utf-8")) 122 | b.JSONBody(matchers.MapMatcher{ 123 | "message": matchers.Like("Not found"), 124 | }) 125 | }). 126 | ExecuteTest(t, func(config consumer.MockServerConfig) error { 127 | url := fmt.Sprintf("http://%s:%d", config.Host, config.Port) 128 | 129 | cfg := ClientConfig{ 130 | AccessToken: "asdf1234", 131 | OrganizationSlug: "buildkite", 132 | ServerBaseUrl: url, 133 | } 134 | 135 | c := NewClient(cfg) 136 | 137 | got, err := c.FetchTestPlan(context.Background(), "rspec", "abc123", 0) 138 | 139 | if err != nil { 140 | t.Errorf("FetchTestPlan() error = %v", err) 141 | } 142 | 143 | if got != nil { 144 | t.Errorf("FetchTestPlan() = %v, want nil", got) 145 | } 146 | 147 | return nil 148 | }) 149 | 150 | if err != nil { 151 | t.Error("mockProvider error", err) 152 | } 153 | } 154 | 155 | func TestFetchTestPlan_BadRequest(t *testing.T) { 156 | requestCount := 0 157 | svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 158 | requestCount++ 159 | http.Error(w, `{"message": "bad request"}`, http.StatusBadRequest) 160 | })) 161 | defer svr.Close() 162 | 163 | cfg := ClientConfig{ 164 | AccessToken: "asdf1234", 165 | OrganizationSlug: "my-org", 166 | ServerBaseUrl: svr.URL, 167 | } 168 | 169 | c := NewClient(cfg) 170 | got, err := c.FetchTestPlan(context.Background(), "my-suite", "xyz", 0) 171 | 172 | if requestCount > 1 { 173 | t.Errorf("http request count = %v, want %d", requestCount, 1) 174 | } 175 | 176 | if err.Error() != "bad request" { 177 | t.Errorf("FetchTestPlan() error = %v, want %v", err, "bad request") 178 | } 179 | 180 | if got != nil { 181 | t.Errorf("FetchTestPlan() = %v, want nil", got) 182 | } 183 | } 184 | 185 | func TestFetchTestPlan_InternalServerError(t *testing.T) { 186 | originalTimeout := retryTimeout 187 | retryTimeout = 1 * time.Millisecond 188 | t.Cleanup(func() { 189 | retryTimeout = originalTimeout 190 | }) 191 | 192 | svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 193 | w.WriteHeader(http.StatusInternalServerError) 194 | })) 195 | defer svr.Close() 196 | 197 | cfg := ClientConfig{ 198 | AccessToken: "asdf1234", 199 | OrganizationSlug: "my-org", 200 | ServerBaseUrl: svr.URL, 201 | } 202 | 203 | c := NewClient(cfg) 204 | got, err := c.FetchTestPlan(context.Background(), "my-suite", "xyz", 0) 205 | 206 | if !errors.Is(err, ErrRetryTimeout) { 207 | t.Errorf("FetchTestPlan() error = %v, want %v", err, ErrRetryTimeout) 208 | } 209 | 210 | if got != nil { 211 | t.Errorf("FetchTestPlan() = %v, want nil", got) 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /internal/api/filter_tests.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/buildkite/test-engine-client/internal/plan" 9 | ) 10 | 11 | type FilterTestsParams struct { 12 | Files []plan.TestCase `json:"files"` 13 | Env map[string]string `json:"env"` 14 | } 15 | 16 | type FilteredTest struct { 17 | Path string `json:"path"` 18 | } 19 | 20 | type filteredTestResponse struct { 21 | Tests []FilteredTest `json:"tests"` 22 | } 23 | 24 | // FilterTests filters tests from the server. It returns a list of tests that need to be split by example. 25 | // Currently, it only filters tests that are slow. 26 | func (c Client) FilterTests(ctx context.Context, suiteSlug string, params FilterTestsParams) ([]FilteredTest, error) { 27 | url := fmt.Sprintf("%s/v2/analytics/organizations/%s/suites/%s/test_plan/filter_tests", c.ServerBaseUrl, c.OrganizationSlug, suiteSlug) 28 | 29 | var response filteredTestResponse 30 | _, err := c.DoWithRetry(ctx, httpRequest{ 31 | Method: http.MethodPost, 32 | URL: url, 33 | Body: params, 34 | }, &response) 35 | 36 | if err != nil { 37 | return []FilteredTest{}, err 38 | } 39 | 40 | return response.Tests, nil 41 | } 42 | -------------------------------------------------------------------------------- /internal/api/filter_tests_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | "time" 11 | 12 | "github.com/buildkite/test-engine-client/internal/plan" 13 | "github.com/google/go-cmp/cmp" 14 | "github.com/pact-foundation/pact-go/v2/consumer" 15 | "github.com/pact-foundation/pact-go/v2/matchers" 16 | ) 17 | 18 | func TestFilterTests_SlowFiles(t *testing.T) { 19 | mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{ 20 | Consumer: "TestEngineClient", 21 | Provider: "TestEngineServer", 22 | }) 23 | 24 | if err != nil { 25 | t.Error("Error mocking provider", err) 26 | } 27 | 28 | params := FilterTestsParams{ 29 | Files: []plan.TestCase{ 30 | { 31 | Path: "./cat_spec.rb", 32 | }, 33 | { 34 | Path: "./dog_spec.rb", 35 | }, 36 | { 37 | Path: "./turtle_spec.rb", 38 | }, 39 | }, 40 | Env: map[string]string{ 41 | "BUILDKITE_PARALLEL_JOB_COUNT": "3", 42 | "BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE": "true", 43 | }, 44 | } 45 | 46 | err = mockProvider. 47 | AddInteraction(). 48 | Given("A slow file exists"). 49 | UponReceiving("A request to filter tests"). 50 | WithRequest("POST", "/v2/analytics/organizations/buildkite/suites/rspec/test_plan/filter_tests", func(b *consumer.V2RequestBuilder) { 51 | b.Header("Authorization", matchers.Like("Bearer asdf1234")) 52 | b.JSONBody(params) 53 | }). 54 | WillRespondWith(200, func(b *consumer.V2ResponseBuilder) { 55 | b.Header("Content-Type", matchers.Like("application/json; charset=utf-8")) 56 | b.JSONBody(matchers.MapMatcher{ 57 | "tests": matchers.EachLike(matchers.MapMatcher{ 58 | "path": matchers.Like("./turtle_spec.rb"), 59 | }, 1), 60 | }) 61 | }). 62 | ExecuteTest(t, func(config consumer.MockServerConfig) error { 63 | url := fmt.Sprintf("http://%s:%d", config.Host, config.Port) 64 | c := NewClient(ClientConfig{ 65 | AccessToken: "asdf1234", 66 | OrganizationSlug: "buildkite", 67 | ServerBaseUrl: url, 68 | }) 69 | got, err := c.FilterTests(context.Background(), "rspec", params) 70 | if err != nil { 71 | t.Errorf("FilterTests() error = %v", err) 72 | } 73 | want := []FilteredTest{ 74 | { 75 | Path: "./turtle_spec.rb", 76 | }, 77 | } 78 | 79 | if diff := cmp.Diff(got, want); diff != "" { 80 | t.Errorf("FilterTests() diff (-got +want):\n%s", diff) 81 | } 82 | return nil 83 | }) 84 | 85 | if err != nil { 86 | t.Error(err) 87 | } 88 | } 89 | 90 | func TestFilterTests_InternalServerError(t *testing.T) { 91 | originalTimeout := retryTimeout 92 | retryTimeout = 1 * time.Millisecond 93 | t.Cleanup(func() { 94 | retryTimeout = originalTimeout 95 | }) 96 | 97 | svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 98 | http.Error(w, `{"message": "something went wrong"}`, http.StatusInternalServerError) 99 | })) 100 | defer svr.Close() 101 | 102 | c := NewClient(ClientConfig{ 103 | OrganizationSlug: "msy-org", 104 | ServerBaseUrl: svr.URL, 105 | }) 106 | 107 | _, err := c.FilterTests(context.Background(), "my-suite", FilterTestsParams{ 108 | Files: []plan.TestCase{}, 109 | }) 110 | 111 | if !errors.Is(err, ErrRetryTimeout) { 112 | t.Errorf("FilterTests() error = %v, want %v", err, ErrRetryTimeout) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /internal/api/post_test_plan_metadata.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/buildkite/test-engine-client/internal/runner" 9 | ) 10 | 11 | type Timeline struct { 12 | Timestamp string `json:"timestamp"` 13 | Event string `json:"event"` 14 | } 15 | 16 | type TestPlanMetadataParams struct { 17 | Version string `json:"version"` 18 | Env map[string]string `json:"env"` 19 | Timeline []Timeline `json:"timeline"` 20 | Statistics runner.RunStatistics `json:"statistics"` 21 | } 22 | 23 | func (c Client) PostTestPlanMetadata(ctx context.Context, suiteSlug string, identifier string, params TestPlanMetadataParams) error { 24 | url := fmt.Sprintf("%s/v2/analytics/organizations/%s/suites/%s/test_plan_metadata", c.ServerBaseUrl, c.OrganizationSlug, suiteSlug) 25 | 26 | _, err := c.DoWithRetry(ctx, httpRequest{ 27 | Method: http.MethodPost, 28 | URL: url, 29 | Body: params, 30 | }, nil) 31 | 32 | return err 33 | } 34 | -------------------------------------------------------------------------------- /internal/api/post_test_plan_metadata_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/buildkite/test-engine-client/internal/runner" 9 | "github.com/pact-foundation/pact-go/v2/consumer" 10 | "github.com/pact-foundation/pact-go/v2/matchers" 11 | ) 12 | 13 | func TestPostTestPlanMetadata(t *testing.T) { 14 | mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{ 15 | Consumer: "TestEngineClient", 16 | Provider: "TestEngineServer", 17 | }) 18 | 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | params := TestPlanMetadataParams{ 24 | Version: "0.7.0", 25 | Env: map[string]string{ 26 | "BUILDKITE_PARALLEL_JOB_COUNT": "3", 27 | "BUILDKITE_PARALLEL_JOB": "1", 28 | "BUILDKITE_TEST_ENGINE_SUITE_SLUG": "my_slug", 29 | "BUILDKITE_TEST_ENGINE_TEST_EXCLUDE_PATTERN": "", 30 | "BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE": "false", 31 | "BUILDKITE_TEST_ENGINE_IDENTIFIER": "abc123", 32 | }, 33 | Timeline: []Timeline{ 34 | { 35 | Event: "test_start", 36 | Timestamp: "2024-06-20T04:46:13.60977Z", 37 | }, 38 | { 39 | Event: "test_end", 40 | Timestamp: "2024-06-20T04:49:09.609793Z", 41 | }, 42 | }, 43 | Statistics: runner.RunStatistics{ 44 | Total: 3, 45 | }, 46 | } 47 | 48 | err = mockProvider. 49 | AddInteraction(). 50 | Given("A test plan exists"). 51 | UponReceiving("A request to post test plan metadata with identifier abc123"). 52 | WithRequest("POST", "/v2/analytics/organizations/buildkite/suites/rspec/test_plan_metadata", func(b *consumer.V2RequestBuilder) { 53 | b.Header("Authorization", matchers.String("Bearer asdf1234")) 54 | b.Header("Content-Type", matchers.String("application/json")) 55 | b.JSONBody(params) 56 | }). 57 | WillRespondWith(200, func(b *consumer.V2ResponseBuilder) { 58 | b.Header("Content-Type", matchers.Like("application/json; charset=utf-8")) 59 | b.JSONBody(matchers.MapMatcher{ 60 | "head": matchers.String("no_content"), 61 | }) 62 | }). 63 | ExecuteTest(t, func(config consumer.MockServerConfig) error { 64 | url := fmt.Sprintf("http://%s:%d", config.Host, config.Port) 65 | c := NewClient(ClientConfig{ 66 | AccessToken: "asdf1234", 67 | OrganizationSlug: "buildkite", 68 | ServerBaseUrl: url, 69 | }) 70 | 71 | _, err := c.DoWithRetry(context.Background(), httpRequest{ 72 | Method: "POST", 73 | URL: fmt.Sprintf("%s/v2/analytics/organizations/%s/suites/%s/test_plan_metadata", c.ServerBaseUrl, c.OrganizationSlug, "rspec"), 74 | Body: params, 75 | }, nil) 76 | 77 | if err != nil { 78 | t.Errorf("PostTestPlanMetadata() error = %v", err) 79 | } 80 | 81 | return nil 82 | }) 83 | 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | } 88 | 89 | func TestPostTestPlanMetadata_NotFound(t *testing.T) { 90 | mockProvider, err := consumer.NewV2Pact(consumer.MockHTTPProviderConfig{ 91 | Consumer: "TestEngineClient", 92 | Provider: "TestEngineServer", 93 | }) 94 | 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | 99 | params := TestPlanMetadataParams{ 100 | Version: "0.7.0", 101 | Env: map[string]string{ 102 | "BUILDKITE_PARALLEL_JOB_COUNT": "3", 103 | "BUILDKITE_PARALLEL_JOB": "1", 104 | "BUILDKITE_TEST_ENGINE_SUITE_SLUG": "my_slug", 105 | "BUILDKITE_TEST_ENGINE_TEST_EXCLUDE_PATTERN": "", 106 | "BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE": "false", 107 | "BUILDKITE_TEST_ENGINE_IDENTIFIER": "abc123", 108 | }, 109 | Timeline: []Timeline{ 110 | { 111 | Event: "test_start", 112 | Timestamp: "2024-06-20T04:46:13.60977Z", 113 | }, 114 | { 115 | Event: "test_end", 116 | Timestamp: "2024-06-20T04:49:09.609793Z", 117 | }, 118 | }, 119 | Statistics: runner.RunStatistics{ 120 | Total: 3, 121 | }, 122 | } 123 | 124 | err = mockProvider. 125 | AddInteraction(). 126 | Given("A test plan doesn't exist"). 127 | UponReceiving("A request to post test plan metadata with identifier abc123"). 128 | WithRequest("POST", "/v2/analytics/organizations/buildkite/suites/rspec/test_plan_metadata", func(b *consumer.V2RequestBuilder) { 129 | b.Header("Authorization", matchers.String("Bearer asdf1234")) 130 | b.Header("Content-Type", matchers.String("application/json")) 131 | b.JSONBody(params) 132 | }). 133 | WillRespondWith(404, func(b *consumer.V2ResponseBuilder) { 134 | b.Header("Content-Type", matchers.Like("application/json; charset=utf-8")) 135 | b.JSONBody(matchers.MapMatcher{ 136 | "message": matchers.Like("Test plan not found"), 137 | }) 138 | }). 139 | ExecuteTest(t, func(config consumer.MockServerConfig) error { 140 | url := fmt.Sprintf("http://%s:%d", config.Host, config.Port) 141 | c := NewClient(ClientConfig{ 142 | AccessToken: "asdf1234", 143 | OrganizationSlug: "buildkite", 144 | ServerBaseUrl: url, 145 | }) 146 | 147 | _, err := c.DoWithRetry(context.Background(), httpRequest{ 148 | Method: "POST", 149 | URL: fmt.Sprintf("%s/v2/analytics/organizations/%s/suites/%s/test_plan_metadata", c.ServerBaseUrl, c.OrganizationSlug, "rspec"), 150 | Body: params, 151 | }, nil) 152 | 153 | if err == nil { 154 | t.Errorf("PostTestPlanMetadata() error = %v, want %v", err, "Test plan not found") 155 | } 156 | 157 | return nil 158 | }) 159 | 160 | if err != nil { 161 | t.Fatal(err) 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "github.com/buildkite/test-engine-client/internal/env" 4 | 5 | // Config is the internal representation of the complete test engine client configuration. 6 | type Config struct { 7 | // AccessToken is the access token for the API. 8 | AccessToken string 9 | // Identifier is the identifier of the build. 10 | Identifier string 11 | // MaxRetries is the maximum number of retries for a failed test. 12 | MaxRetries int 13 | // RetryCommand is the command to run the retry tests. 14 | RetryCommand string 15 | // Node index is index of the current node. 16 | NodeIndex int 17 | // OrganizationSlug is the slug of the organization. 18 | OrganizationSlug string 19 | // Parallelism is the number of parallel tasks to run. 20 | Parallelism int 21 | // The path to the result file. 22 | ResultPath string 23 | // Whether a failed muted test should be retried. 24 | // This is default to true because we want more signal for our flaky detection system. 25 | RetryForMutedTest bool 26 | // ServerBaseUrl is the base URL of the test plan server. 27 | ServerBaseUrl string 28 | // SplitByExample is the flag to enable split the test by example. 29 | SplitByExample bool 30 | // SuiteSlug is the slug of the suite. 31 | SuiteSlug string 32 | // TestCommand is the command to run the tests. 33 | TestCommand string 34 | // TestFilePattern is the pattern to match the test files. 35 | TestFilePattern string 36 | // TestFileExcludePattern is the pattern to exclude the test files. 37 | TestFileExcludePattern string 38 | // TestRunner is the name of the runner. 39 | TestRunner string 40 | // Branch is the string value of the git branch name, used by Buildkite only. 41 | Branch string 42 | // JobRetryCount is the count of the number of times the job has been retried. 43 | JobRetryCount int 44 | // Env provides access to environment variables. 45 | // It's public because many tests in other packages reference it (perhaps they should not). 46 | Env env.Env 47 | // errs is a map of environment variables name and the validation errors associated with them. 48 | errs InvalidConfigError 49 | } 50 | 51 | // New wraps the readFromEnv and validate functions to create a new Config struct. 52 | // It returns Config struct and an InvalidConfigError if there is an invalid configuration. 53 | func New(env env.Env) (Config, error) { 54 | c := Config{ 55 | Env: env, 56 | errs: InvalidConfigError{}, 57 | } 58 | 59 | // TODO: remove error from readFromEnv and validate functions 60 | _ = c.readFromEnv(env) 61 | _ = c.validate() 62 | 63 | if len(c.errs) > 0 { 64 | return Config{}, c.errs 65 | } 66 | 67 | return c, nil 68 | } 69 | -------------------------------------------------------------------------------- /internal/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/buildkite/test-engine-client/internal/env" 8 | "github.com/google/go-cmp/cmp" 9 | "github.com/google/go-cmp/cmp/cmpopts" 10 | ) 11 | 12 | func getExampleEnv() env.Env { 13 | return env.Map{ 14 | "BUILDKITE_PARALLEL_JOB_COUNT": "60", 15 | "BUILDKITE_PARALLEL_JOB": "7", 16 | "BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN": "my_token", 17 | "BUILDKITE_TEST_ENGINE_BASE_URL": "https://build.kite", 18 | "BUILDKITE_TEST_ENGINE_TEST_CMD": "bin/rspec {{testExamples}}", 19 | "BUILDKITE_ORGANIZATION_SLUG": "my_org", 20 | "BUILDKITE_TEST_ENGINE_SUITE_SLUG": "my_suite", 21 | "BUILDKITE_BUILD_ID": "123", 22 | "BUILDKITE_STEP_ID": "456", 23 | "BUILDKITE_TEST_ENGINE_TEST_RUNNER": "rspec", 24 | "BUILDKITE_TEST_ENGINE_DISABLE_RETRY_FOR_MUTED_TEST": "true", 25 | "BUILDKITE_TEST_ENGINE_RESULT_PATH": "tmp/rspec.json", 26 | "BUILDKITE_RETRY_COUNT": "0", 27 | } 28 | } 29 | 30 | func TestNewConfig(t *testing.T) { 31 | env := getExampleEnv() 32 | 33 | c, err := New(env) 34 | if err != nil { 35 | t.Errorf("config.New() error = %v", err) 36 | } 37 | 38 | want := Config{ 39 | Parallelism: 60, 40 | NodeIndex: 7, 41 | ServerBaseUrl: "https://build.kite", 42 | Identifier: "123/456", 43 | TestCommand: "bin/rspec {{testExamples}}", 44 | AccessToken: "my_token", 45 | OrganizationSlug: "my_org", 46 | RetryForMutedTest: false, 47 | ResultPath: "tmp/rspec.json", 48 | SuiteSlug: "my_suite", 49 | TestRunner: "rspec", 50 | JobRetryCount: 0, 51 | Env: env, 52 | errs: InvalidConfigError{}, 53 | } 54 | 55 | if diff := cmp.Diff(c, want, cmpopts.IgnoreUnexported(Config{})); diff != "" { 56 | t.Errorf("config.New() diff (-got +want):\n%s", diff) 57 | } 58 | } 59 | 60 | func TestNewConfig_EmptyConfig(t *testing.T) { 61 | _, err := New(env.Map{}) 62 | 63 | if !errors.As(err, new(InvalidConfigError)) { 64 | t.Errorf("config.Validate() error = %v, want InvalidConfigError", err) 65 | } 66 | } 67 | 68 | func TestNewConfig_MissingConfigWithDefault(t *testing.T) { 69 | env := getExampleEnv() 70 | env.Delete("BUILDKITE_TEST_ENGINE_MODE") 71 | env.Delete("BUILDKITE_TEST_ENGINE_BASE_URL") 72 | env.Delete("BUILDKITE_TEST_ENGINE_TEST_CMD") 73 | env.Delete("BUILDKITE_TEST_ENGINE_DISABLE_RETRY_FOR_MUTED_TEST") 74 | 75 | c, err := New(env) 76 | if err != nil { 77 | t.Errorf("config.New() error = %v", err) 78 | } 79 | 80 | want := Config{ 81 | Parallelism: 60, 82 | NodeIndex: 7, 83 | ServerBaseUrl: "https://api.buildkite.com", 84 | Identifier: "123/456", 85 | AccessToken: "my_token", 86 | OrganizationSlug: "my_org", 87 | SuiteSlug: "my_suite", 88 | TestRunner: "rspec", 89 | RetryForMutedTest: true, 90 | ResultPath: "tmp/rspec.json", 91 | JobRetryCount: 0, 92 | Env: env, 93 | } 94 | 95 | if diff := cmp.Diff(c, want, cmpopts.IgnoreUnexported(Config{})); diff != "" { 96 | t.Errorf("config.New() diff (-got +want):\n%s", diff) 97 | } 98 | } 99 | 100 | func TestNewConfig_InvalidConfig(t *testing.T) { 101 | env := getExampleEnv() 102 | env.Set("BUILDKITE_TEST_ENGINE_MODE", "dynamic") 103 | env.Delete("BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN") 104 | 105 | _, err := New(env) 106 | 107 | var invConfigError InvalidConfigError 108 | if !errors.As(err, &invConfigError) { 109 | t.Errorf("config.Validate() error = %v, want InvalidConfigError", err) 110 | } 111 | 112 | if len(invConfigError) != 1 { 113 | t.Errorf("config.readFromEnv() error length = %d, want 2", len(invConfigError)) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /internal/config/doc.go: -------------------------------------------------------------------------------- 1 | // Package config provides the configuration for the test engine client. 2 | package config 3 | -------------------------------------------------------------------------------- /internal/config/env.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/buildkite/test-engine-client/internal/env" 7 | ) 8 | 9 | // getEnvWithDefault retrieves the value of the environment variable named by the key. 10 | // If the variable is present and not empty, the value is returned. 11 | // Otherwise the returned value will be the default value. 12 | func getEnvWithDefault(env env.Env, key string, defaultValue string) string { 13 | value, ok := env.Lookup(key) 14 | if !ok { 15 | return defaultValue 16 | } 17 | if value == "" { 18 | return defaultValue 19 | } 20 | return value 21 | } 22 | 23 | func getIntEnvWithDefault(env env.Env, key string, defaultValue int) (int, error) { 24 | value := env.Get(key) 25 | // If the environment variable is not set, return the default value. 26 | if value == "" { 27 | return defaultValue, nil 28 | } 29 | // Convert the value to int, and return error if it fails. 30 | valueInt, err := strconv.Atoi(value) 31 | if err != nil { 32 | return defaultValue, err 33 | } 34 | // Return the value if it's successfully converted to int. 35 | return valueInt, nil 36 | } 37 | 38 | func (c Config) DumpEnv() map[string]string { 39 | keys := []string{ 40 | "BUILDKITE_BUILD_ID", 41 | "BUILDKITE_JOB_ID", 42 | "BUILDKITE_ORGANIZATION_SLUG", 43 | "BUILDKITE_PARALLEL_JOB_COUNT", 44 | "BUILDKITE_PARALLEL_JOB", 45 | "BUILDKITE_TEST_ENGINE_DEBUG_ENABLED", 46 | "BUILDKITE_TEST_ENGINE_RETRY_COUNT", 47 | "BUILDKITE_TEST_ENGINE_RETRY_CMD", 48 | "BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE", 49 | "BUILDKITE_TEST_ENGINE_SUITE_SLUG", 50 | "BUILDKITE_TEST_ENGINE_TEST_CMD", 51 | "BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN", 52 | "BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN", 53 | "BUILDKITE_TEST_ENGINE_TEST_RUNNER", 54 | "BUILDKITE_STEP_ID", 55 | "BUILDKITE_BRANCH", 56 | "BUILDKITE_RETRY_COUNT", 57 | } 58 | 59 | envs := make(map[string]string) 60 | for _, key := range keys { 61 | envs[key] = c.Env.Get(key) 62 | } 63 | 64 | envs["BUILDKITE_TEST_ENGINE_IDENTIFIER"] = c.Identifier 65 | 66 | return envs 67 | } 68 | -------------------------------------------------------------------------------- /internal/config/env_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/buildkite/test-engine-client/internal/env" 9 | ) 10 | 11 | func TestGetIntEnvWithDefault(t *testing.T) { 12 | env := env.Map{ 13 | "MY_KEY": "10", 14 | "EMPTY_KEY": "", 15 | "INVALID_KEY": "invalid_value", 16 | } 17 | 18 | tests := []struct { 19 | key string 20 | defaultValue int 21 | want int 22 | err error 23 | }{ 24 | { 25 | key: "MY_KEY", 26 | defaultValue: 20, 27 | want: 10, 28 | err: nil, 29 | }, 30 | { 31 | key: "NON_EXISTENT_KEY", 32 | defaultValue: 30, 33 | want: 30, 34 | err: nil, 35 | }, 36 | { 37 | key: "EMPTY_KEY", 38 | defaultValue: 40, 39 | want: 40, 40 | err: nil, 41 | }, 42 | { 43 | key: "INVALID_KEY", 44 | defaultValue: 50, 45 | want: 50, 46 | err: strconv.ErrSyntax, 47 | }, 48 | } 49 | 50 | for _, tt := range tests { 51 | t.Run(tt.key, func(t *testing.T) { 52 | got, err := getIntEnvWithDefault(env, tt.key, tt.defaultValue) 53 | if err != nil && !errors.Is(err, tt.err) { 54 | t.Errorf("getIntEnvWithDefault(%q, %d) error = %v, want %v", tt.key, tt.defaultValue, err, tt.err) 55 | } 56 | if got != tt.want { 57 | t.Errorf("getIntEnvWithDefault(%q, %d) = %d, want %d", tt.key, tt.defaultValue, got, tt.want) 58 | } 59 | }) 60 | } 61 | } 62 | 63 | func TestGetEnvWithDefault(t *testing.T) { 64 | env := env.Map{ 65 | "MY_KEY": "my_value", 66 | "EMPTY_KEY": "", 67 | "OTHER_KEY": "other_value", 68 | } 69 | 70 | tests := []struct { 71 | key string 72 | defaultValue string 73 | want string 74 | }{ 75 | { 76 | key: "MY_KEY", 77 | defaultValue: "default_value", 78 | want: "my_value", 79 | }, 80 | { 81 | key: "NON_EXISTENT_KEY", 82 | defaultValue: "non_existent_default_value", 83 | want: "non_existent_default_value", 84 | }, 85 | { 86 | key: "EMPTY_KEY", 87 | defaultValue: "empty_default_value", 88 | want: "empty_default_value", 89 | }, 90 | { 91 | key: "EMPTY_KEY", 92 | defaultValue: env.Get("OTHER_KEY"), 93 | want: "other_value", 94 | }, 95 | } 96 | 97 | for _, tt := range tests { 98 | t.Run(tt.key, func(t *testing.T) { 99 | if got := getEnvWithDefault(env, tt.key, tt.defaultValue); got != tt.want { 100 | t.Errorf("getEnvWithDefault(%q, %q) = %q, want %q", tt.key, tt.defaultValue, got, tt.want) 101 | } 102 | }) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /internal/config/error.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | ) 8 | 9 | // InvalidConfigError is an error that contains a map of all validation errors on each field of the configuration. 10 | type InvalidConfigError map[string][]error 11 | 12 | func (i InvalidConfigError) Error() string { 13 | var errs []string 14 | for field, value := range i { 15 | for _, v := range value { 16 | errs = append(errs, fmt.Sprintf("%s %s", field, v)) 17 | } 18 | } 19 | sort.Strings(errs) 20 | return strings.Join(errs, "\n") 21 | } 22 | 23 | func (e InvalidConfigError) appendFieldError(field, format string, v ...any) { 24 | if e[field] == nil { 25 | e[field] = make([]error, 0) 26 | } 27 | e[field] = append(e[field], fmt.Errorf(format, v...)) 28 | } 29 | -------------------------------------------------------------------------------- /internal/config/read.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/buildkite/test-engine-client/internal/env" 9 | ) 10 | 11 | // readFromEnv reads the configuration from environment variables and sets it to the Config struct. 12 | // It returns an InvalidConfigError if there is an error while reading the configuration 13 | // such as when parallelism and node index are not numbers, and set a default 14 | // value for ServerBaseUrl if they are not set. 15 | // 16 | // Currently, it reads the following environment variables: 17 | // - BUILDKITE_ORGANIZATION_SLUG (OrganizationSlug) 18 | // - BUILDKITE_PARALLEL_JOB_COUNT (Parallelism) 19 | // - BUILDKITE_PARALLEL_JOB (NodeIndex) 20 | // - BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN (AccessToken) 21 | // - BUILDKITE_TEST_ENGINE_BASE_URL (ServerBaseUrl) 22 | // - BUILDKITE_TEST_ENGINE_RETRY_COUNT (MaxRetries) 23 | // - BUILDKITE_TEST_ENGINE_RETRY_CMD (RetryCommand) 24 | // - BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE (SplitByExample) 25 | // - BUILDKITE_TEST_ENGINE_SUITE_SLUG (SuiteSlug) 26 | // - BUILDKITE_TEST_ENGINE_TEST_CMD (TestCommand) 27 | // - BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN (TestFilePattern) 28 | // - BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN (TestFileExcludePattern) 29 | // - BUILDKITE_BRANCH (Branch) 30 | // - BUILDKITE_RETRY_COUNT (JobRetryCount) 31 | // 32 | // If we are going to support other CI environment in the future, 33 | // we will need to change where we read the configuration from. 34 | func (c *Config) readFromEnv(env env.Env) error { 35 | c.AccessToken = env.Get("BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN") 36 | c.OrganizationSlug = env.Get("BUILDKITE_ORGANIZATION_SLUG") 37 | c.SuiteSlug = env.Get("BUILDKITE_TEST_ENGINE_SUITE_SLUG") 38 | 39 | buildId := env.Get("BUILDKITE_BUILD_ID") 40 | if buildId == "" { 41 | c.errs.appendFieldError("BUILDKITE_BUILD_ID", "must not be blank") 42 | } 43 | 44 | stepId := env.Get("BUILDKITE_STEP_ID") 45 | if stepId == "" { 46 | c.errs.appendFieldError("BUILDKITE_STEP_ID", "must not be blank") 47 | } 48 | 49 | c.Identifier = fmt.Sprintf("%s/%s", buildId, stepId) 50 | 51 | c.ServerBaseUrl = getEnvWithDefault(env, "BUILDKITE_TEST_ENGINE_BASE_URL", "https://api.buildkite.com") 52 | c.TestCommand = env.Get("BUILDKITE_TEST_ENGINE_TEST_CMD") 53 | c.TestFilePattern = env.Get("BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN") 54 | c.TestFileExcludePattern = env.Get("BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN") 55 | c.TestRunner = env.Get("BUILDKITE_TEST_ENGINE_TEST_RUNNER") 56 | c.RetryForMutedTest = strings.ToLower(env.Get("BUILDKITE_TEST_ENGINE_DISABLE_RETRY_FOR_MUTED_TEST")) != "true" 57 | c.ResultPath = env.Get("BUILDKITE_TEST_ENGINE_RESULT_PATH") 58 | 59 | c.SplitByExample = strings.ToLower(env.Get("BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE")) == "true" 60 | 61 | // used by Buildkite only, for experimental plans 62 | c.Branch = env.Get("BUILDKITE_BRANCH") 63 | 64 | JobRetryCount, err := getIntEnvWithDefault(env, "BUILDKITE_RETRY_COUNT", 0) 65 | c.JobRetryCount = JobRetryCount 66 | if err != nil { 67 | c.errs.appendFieldError("BUILDKITE_RETRY_COUNT", "was %q, must be a number", env.Get("BUILDKITE_RETRY_COUNT")) 68 | } 69 | 70 | MaxRetries, err := getIntEnvWithDefault(env, "BUILDKITE_TEST_ENGINE_RETRY_COUNT", 0) 71 | c.MaxRetries = MaxRetries 72 | if err != nil { 73 | c.errs.appendFieldError("BUILDKITE_TEST_ENGINE_RETRY_COUNT", "was %q, must be a number", env.Get("BUILDKITE_TEST_ENGINE_RETRY_COUNT")) 74 | } 75 | c.RetryCommand = env.Get("BUILDKITE_TEST_ENGINE_RETRY_CMD") 76 | 77 | parallelism := env.Get("BUILDKITE_PARALLEL_JOB_COUNT") 78 | parallelismInt, err := strconv.Atoi(parallelism) 79 | if err != nil { 80 | c.errs.appendFieldError("BUILDKITE_PARALLEL_JOB_COUNT", "was %q, must be a number", parallelism) 81 | } 82 | c.Parallelism = parallelismInt 83 | 84 | nodeIndex := env.Get("BUILDKITE_PARALLEL_JOB") 85 | nodeIndexInt, err := strconv.Atoi(nodeIndex) 86 | if err != nil { 87 | c.errs.appendFieldError("BUILDKITE_PARALLEL_JOB", "was %q, must be a number", nodeIndex) 88 | } 89 | c.NodeIndex = nodeIndexInt 90 | 91 | if len(c.errs) > 0 { 92 | return c.errs 93 | } 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /internal/config/validate.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "net/url" 5 | ) 6 | 7 | // validate checks if the Config struct is valid and returns InvalidConfigError if it's invalid. 8 | func (c *Config) validate() error { 9 | 10 | if c.MaxRetries < 0 { 11 | c.errs.appendFieldError("BUILDKITE_TEST_ENGINE_RETRY_COUNT", "was %d, must be greater than or equal to 0", c.MaxRetries) 12 | } 13 | 14 | // We validate BUILDKITE_PARALLEL_JOB and BUILDKITE_PARALLEL_JOB_COUNT in two steps. 15 | // 1. Validate the type and presence of BUILDKITE_PARALLEL_JOB and BUILDKITE_PARALLEL_JOB_COUNT when reading them from the environment. See readFromEnv() in ./read.go. 16 | // 2. Validate the range of BUILDKITE_PARALLEL_JOB and BUILDKITE_PARALLEL_JOB_COUNT 17 | // 18 | // This is the second step. We don't validate the range of BUILDKITE_PARALLEL_JOB and BUILDKITE_PARALLEL_JOB_COUNT if the first validation step fails. 19 | // 20 | // The order of the range validation matters. 21 | // The range validation of BUILDKITE_PARALLEL_JOB depends on the result of BUILDKITE_PARALLEL_JOB_COUNT validation at the first step. 22 | // We need to validate the range of BUILDKITE_PARALLEL_JOB first before we add the range validation error to BUILDKITE_PARALLEL_JOB_COUNT. 23 | if c.errs["BUILDKITE_PARALLEL_JOB"] == nil { 24 | if got, min := c.NodeIndex, 0; got < 0 { 25 | c.errs.appendFieldError("BUILDKITE_PARALLEL_JOB", "was %d, must be greater than or equal to %d", got, min) 26 | } 27 | 28 | if c.errs["BUILDKITE_PARALLEL_JOB_COUNT"] == nil { 29 | if got, max := c.NodeIndex, c.Parallelism-1; got > max { 30 | c.errs.appendFieldError("BUILDKITE_PARALLEL_JOB", "was %d, must not be greater than %d", got, max) 31 | } 32 | } 33 | } 34 | 35 | if c.errs["BUILDKITE_PARALLEL_JOB_COUNT"] == nil { 36 | if got, min := c.Parallelism, 1; got < min { 37 | c.errs.appendFieldError("BUILDKITE_PARALLEL_JOB_COUNT", "was %d, must be greater than or equal to %d", got, min) 38 | } 39 | 40 | if got, max := c.Parallelism, 1000; got > max { 41 | c.errs.appendFieldError("BUILDKITE_PARALLEL_JOB_COUNT", "was %d, must not be greater than %d", got, max) 42 | } 43 | } 44 | 45 | if c.ServerBaseUrl != "" { 46 | if _, err := url.ParseRequestURI(c.ServerBaseUrl); err != nil { 47 | c.errs.appendFieldError("BUILDKITE_TEST_ENGINE_BASE_URL", "must be a valid URL") 48 | } 49 | } 50 | 51 | if c.AccessToken == "" { 52 | c.errs.appendFieldError("BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN", "must not be blank") 53 | } 54 | 55 | if c.OrganizationSlug == "" { 56 | c.errs.appendFieldError("BUILDKITE_ORGANIZATION_SLUG", "must not be blank") 57 | } 58 | 59 | if c.SuiteSlug == "" { 60 | c.errs.appendFieldError("BUILDKITE_TEST_ENGINE_SUITE_SLUG", "must not be blank") 61 | } 62 | 63 | if c.ResultPath == "" && c.TestRunner != "cypress" && c.TestRunner != "pytest" && c.TestRunner != "pytest-pants" { 64 | c.errs.appendFieldError("BUILDKITE_TEST_ENGINE_RESULT_PATH", "must not be blank") 65 | } 66 | 67 | if c.TestRunner == "" { 68 | c.errs.appendFieldError("BUILDKITE_TEST_ENGINE_TEST_RUNNER", "must not be blank") 69 | } 70 | 71 | if len(c.errs) > 0 { 72 | return c.errs 73 | } 74 | 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /internal/config/validate_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | ) 7 | 8 | func createConfig() Config { 9 | return Config{ 10 | ServerBaseUrl: "http://example.com", 11 | Parallelism: 10, 12 | NodeIndex: 0, 13 | Identifier: "my_identifier", 14 | OrganizationSlug: "my_org", 15 | SuiteSlug: "my_suite", 16 | AccessToken: "my_token", 17 | MaxRetries: 3, 18 | ResultPath: "tmp/result-*.json", 19 | errs: InvalidConfigError{}, 20 | TestRunner: "rspec", 21 | } 22 | } 23 | 24 | func TestConfigValidate(t *testing.T) { 25 | c := createConfig() 26 | if err := c.validate(); err != nil { 27 | t.Errorf("config.validate() error = %v", err) 28 | } 29 | } 30 | 31 | func TestConfigValidate_Empty(t *testing.T) { 32 | c := Config{errs: InvalidConfigError{}} 33 | err := c.validate() 34 | 35 | if !errors.As(err, new(InvalidConfigError)) { 36 | t.Errorf("config.validate() error = %v, want InvalidConfigError", err) 37 | } 38 | } 39 | 40 | func TestConfigValidate_Invalid(t *testing.T) { 41 | scenario := []struct { 42 | name string 43 | field string 44 | value any 45 | }{ 46 | // Base URL is bunk 47 | { 48 | name: "BUILDKITE_TEST_ENGINE_BASE_URL", 49 | value: "foo", 50 | }, 51 | // Node index < 0 52 | { 53 | name: "BUILDKITE_PARALLEL_JOB", 54 | value: -1, 55 | }, 56 | // Node index > parallelism 57 | { 58 | name: "BUILDKITE_PARALLEL_JOB", 59 | value: 15, 60 | }, 61 | // Parallelism > 1000 62 | { 63 | name: "BUILDKITE_PARALLEL_JOB_COUNT", 64 | value: 1341, 65 | }, 66 | // Organization slug is missing 67 | { 68 | name: "BUILDKITE_ORGANIZATION_SLUG", 69 | value: "", 70 | }, 71 | // Suite slug is missing 72 | { 73 | name: "BUILDKITE_TEST_ENGINE_SUITE_SLUG", 74 | value: "", 75 | }, 76 | // API access token is blank 77 | { 78 | name: "BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN", 79 | value: "", 80 | }, 81 | // Test runner is blank 82 | { 83 | name: "BUILDKITE_TEST_ENGINE_TEST_RUNNER", 84 | value: "", 85 | }, 86 | } 87 | 88 | for _, s := range scenario { 89 | t.Run(s.name, func(t *testing.T) { 90 | c := createConfig() 91 | switch s.name { 92 | case "BUILDKITE_TEST_ENGINE_BASE_URL": 93 | c.ServerBaseUrl = s.value.(string) 94 | case "BUILDKITE_PARALLEL_JOB": 95 | c.NodeIndex = s.value.(int) 96 | case "BUILDKITE_PARALLEL_JOB_COUNT": 97 | c.Parallelism = s.value.(int) 98 | case "BUILDKITE_ORGANIZATION_SLUG": 99 | c.OrganizationSlug = s.value.(string) 100 | case "BUILDKITE_TEST_ENGINE_SUITE_SLUG": 101 | c.SuiteSlug = s.value.(string) 102 | case "BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN": 103 | c.AccessToken = s.value.(string) 104 | case "BUILDKITE_TEST_ENGINE_TEST_RUNNER": 105 | c.TestRunner = s.value.(string) 106 | } 107 | 108 | err := c.validate() 109 | 110 | var invConfigError InvalidConfigError 111 | if !errors.As(err, &invConfigError) { 112 | t.Errorf("config.validate() error = %v, want InvalidConfigError", err) 113 | } 114 | 115 | if len(invConfigError) != 1 { 116 | t.Errorf("config.validate() error length = %d, want 1", len(invConfigError)) 117 | } 118 | 119 | if len(invConfigError[s.name]) != 1 { 120 | t.Errorf("config.validate() error for %s length = %d, want 1", s.name, len(invConfigError[s.name])) 121 | } 122 | }) 123 | } 124 | 125 | t.Run("Parallelism is less than 1", func(t *testing.T) { 126 | c := createConfig() 127 | c.Parallelism = 0 128 | err := c.validate() 129 | 130 | var invConfigError InvalidConfigError 131 | if !errors.As(err, &invConfigError) { 132 | t.Errorf("config.validate() error = %v, want InvalidConfigError", err) 133 | return 134 | } 135 | 136 | // When parallelism less than 1, node index will always be invalid because it cannot be greater than parallelism and less than 0. 137 | // So, we expect 2 validation errors. 138 | if len(invConfigError) != 2 { 139 | t.Errorf("config.validate() error length = %d, want 2", len(invConfigError)) 140 | } 141 | }) 142 | 143 | t.Run("MaxRetries is less than 0", func(t *testing.T) { 144 | c := createConfig() 145 | c.MaxRetries = -1 146 | err := c.validate() 147 | 148 | var invConfigError InvalidConfigError 149 | if !errors.As(err, &invConfigError) { 150 | t.Errorf("config.validate() error = %v, want InvalidConfigError", err) 151 | return 152 | } 153 | 154 | if len(invConfigError) != 1 { 155 | t.Errorf("config.validate() error length = %d, want 1", len(invConfigError)) 156 | } 157 | }) 158 | } 159 | 160 | func TestConfigValidate_ResultPathOptionalWithCypress(t *testing.T) { 161 | c := createConfig() 162 | c.ResultPath = "" 163 | c.TestRunner = "cypress" 164 | err := c.validate() 165 | 166 | if err != nil { 167 | t.Errorf("config.validate() error = %v", err) 168 | } 169 | } 170 | 171 | func TestConfigValidate_ResultPathOptionalWithPytest(t *testing.T) { 172 | c := createConfig() 173 | c.ResultPath = "" 174 | c.TestRunner = "pytest" 175 | err := c.validate() 176 | 177 | if err != nil { 178 | t.Errorf("config.validate() error = %v", err) 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /internal/debug/debug.go: -------------------------------------------------------------------------------- 1 | package debug 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "os" 7 | ) 8 | 9 | var Enabled = false 10 | var logger = log.New(os.Stdout, "DEBUG: ", log.LstdFlags|log.Lmsgprefix) 11 | 12 | // Printf works like log.Printf, but only when debugging is enabled. 13 | func Printf(format string, v ...interface{}) { 14 | if Enabled { 15 | logger.Printf(format, v...) 16 | } 17 | } 18 | 19 | // Println works like log.Println, but only when debugging is enabled. 20 | func Println(v ...interface{}) { 21 | if Enabled { 22 | logger.Println(v...) 23 | } 24 | } 25 | 26 | // SetDebug sets the debugging state. 27 | // When debugging is enabled (true), debug.Printf and debug.Println will print to output. 28 | // Output by default is os.Stdout, and can be changed with debug.SetOutput. 29 | func SetDebug(b bool) { 30 | Enabled = b 31 | } 32 | 33 | // SetOutput sets the destination for the logger. 34 | // By default, it is set to os.Stdout. 35 | // It can be set to any io.Writer, such as a file or a buffer. 36 | func SetOutput(w io.Writer) { 37 | logger.SetOutput(w) 38 | } 39 | -------------------------------------------------------------------------------- /internal/debug/debug_test.go: -------------------------------------------------------------------------------- 1 | package debug 2 | 3 | import ( 4 | "bytes" 5 | "regexp" 6 | "testing" 7 | ) 8 | 9 | func TestPrintf(t *testing.T) { 10 | var output bytes.Buffer 11 | 12 | SetDebug(true) 13 | SetOutput(&output) 14 | 15 | Printf("Hello, %s!", "world") 16 | 17 | want := "DEBUG: Hello, world!\n" 18 | matched, err := regexp.MatchString(want, output.String()) 19 | if err != nil { 20 | t.Errorf("error matching output: %v", err) 21 | } 22 | 23 | if !matched { 24 | t.Errorf("output doesn't match: got %q, want %q", output.String(), want) 25 | } 26 | } 27 | 28 | func TestPrintf_disabled(t *testing.T) { 29 | var output bytes.Buffer 30 | 31 | SetDebug(false) 32 | SetOutput(&output) 33 | 34 | Printf("Hello, %s!", "world") 35 | if output.String() != "" { 36 | t.Errorf("output should be empty, got %q", output.String()) 37 | } 38 | } 39 | 40 | func TestPrintln(t *testing.T) { 41 | var output bytes.Buffer 42 | 43 | SetDebug(true) 44 | SetOutput(&output) 45 | 46 | Println("Hello world!") 47 | 48 | want := "DEBUG: Hello world!\n" 49 | matched, err := regexp.MatchString(want, output.String()) 50 | if err != nil { 51 | t.Errorf("error matching output: %v", err) 52 | } 53 | 54 | if !matched { 55 | t.Errorf("output doesn't match: got %q, want %q", output.String(), want) 56 | } 57 | } 58 | 59 | func TestPrintln_disabled(t *testing.T) { 60 | var output bytes.Buffer 61 | 62 | SetDebug(false) 63 | SetOutput(&output) 64 | 65 | Println("Hello world!") 66 | if output.String() != "" { 67 | t.Errorf("output should be empty, got %q", output.String()) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /internal/debug/doc.go: -------------------------------------------------------------------------------- 1 | // Package debug provides debugging utilities. 2 | package debug 3 | -------------------------------------------------------------------------------- /internal/env/env.go: -------------------------------------------------------------------------------- 1 | package env 2 | 3 | import "os" 4 | 5 | type Env interface { 6 | Get(key string) string 7 | Set(key string, value string) error 8 | Delete(key string) error 9 | Lookup(key string) (string, bool) 10 | } 11 | 12 | // OS is an Env backed by real operating system environment 13 | type OS struct{} 14 | 15 | func (OS) Get(key string) string { 16 | return os.Getenv(key) 17 | } 18 | 19 | func (OS) Set(key string, value string) error { 20 | return os.Setenv(key, value) 21 | } 22 | 23 | func (OS) Delete(key string) error { 24 | return os.Unsetenv(key) 25 | } 26 | 27 | func (OS) Lookup(key string) (string, bool) { 28 | return os.LookupEnv(key) 29 | } 30 | 31 | // Map is an Env backed by a map[string]string for testing etc 32 | type Map map[string]string 33 | 34 | func (env Map) Get(key string) string { 35 | return env[key] 36 | } 37 | 38 | func (env Map) Set(key string, value string) error { 39 | env[key] = value 40 | return nil 41 | } 42 | 43 | func (env Map) Delete(key string) error { 44 | delete(env, key) 45 | return nil 46 | } 47 | 48 | func (env Map) Lookup(key string) (string, bool) { 49 | if val, ok := env[key]; ok { 50 | return val, true 51 | } else { 52 | return "", false 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /internal/env/env_test.go: -------------------------------------------------------------------------------- 1 | package env_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/buildkite/test-engine-client/internal/env" 8 | "github.com/google/go-cmp/cmp" 9 | ) 10 | 11 | // Note: out of the two implementations of interface env: 12 | // - I'm testing env.OS because it's used by real code, 13 | // - I'm not testing env.Map because it's only used in tests. 14 | 15 | func TestOSGet(t *testing.T) { 16 | defer setenvWithUnset("BKTEC_ENV_TEST_VALUE", "hello")() 17 | 18 | env := env.OS{} 19 | 20 | got, want := env.Get("BKTEC_ENV_TEST_VALUE"), "hello" 21 | if diff := cmp.Diff(got, want); diff != "" { 22 | t.Errorf("env.Get() diff (-got +want):\n%s", diff) 23 | } 24 | } 25 | 26 | func TestOSGetMissing(t *testing.T) { 27 | os.Unsetenv("BKTEC_ENV_TEST_VALUE") // just in case 28 | 29 | env := env.OS{} 30 | 31 | got, want := env.Get("BKTEC_ENV_TEST_VALUE"), "" 32 | if diff := cmp.Diff(got, want); diff != "" { 33 | t.Errorf("env.Get() diff (-got +want):\n%s", diff) 34 | } 35 | } 36 | 37 | func TestOSLookup(t *testing.T) { 38 | defer setenvWithUnset("BKTEC_ENV_TEST_VALUE", "hello")() 39 | 40 | env := env.OS{} 41 | 42 | got, ok := env.Lookup("BKTEC_ENV_TEST_VALUE") 43 | want := "hello" 44 | 45 | if !ok { 46 | t.Errorf("env.Lookup() ok value should be true: %v", ok) 47 | } 48 | 49 | if diff := cmp.Diff(got, want); diff != "" { 50 | t.Errorf("env.Lookup() diff (-got +want):\n%s", diff) 51 | } 52 | } 53 | 54 | func TestOSLookupMissing(t *testing.T) { 55 | os.Unsetenv("BKTEC_ENV_TEST_VALUE") // just in case 56 | 57 | env := env.OS{} 58 | 59 | got, ok := env.Lookup("BKTEC_ENV_TEST_VALUE") 60 | want := "" 61 | 62 | if ok { 63 | t.Errorf("env.Lookup() ok value should be false: %v", ok) 64 | } 65 | 66 | if diff := cmp.Diff(got, want); diff != "" { 67 | t.Errorf("env.Lookup() diff (-got +want):\n%s", diff) 68 | } 69 | } 70 | 71 | func TestOSDelete(t *testing.T) { 72 | defer setenvWithUnset("BKTEC_ENV_TEST_VALUE", "hello")() 73 | 74 | env := env.OS{} 75 | 76 | err := env.Delete("BKTEC_ENV_TEST_VALUE") 77 | if err != nil { 78 | t.Error(err) 79 | } 80 | 81 | got, want := os.Getenv("BKTEC_ENV_TEST_VALUE"), "" 82 | if diff := cmp.Diff(got, want); diff != "" { 83 | t.Errorf("os.Getenv() diff (-got +want):\n%s", diff) 84 | } 85 | } 86 | 87 | func TestOSSet(t *testing.T) { 88 | os.Unsetenv("BKTEC_ENV_TEST_VALUE") // ensure pre-condition 89 | defer os.Unsetenv("BKTEC_ENV_TEST_VALUE") // ensure post-condition (cleanup) 90 | 91 | env := env.OS{} 92 | 93 | err := env.Set("BKTEC_ENV_TEST_VALUE", "Set()") 94 | if err != nil { 95 | t.Error(err) 96 | } 97 | 98 | got, want := os.Getenv("BKTEC_ENV_TEST_VALUE"), "Set()" 99 | if diff := cmp.Diff(got, want); diff != "" { 100 | t.Errorf("os.Getenv() diff (-got +want):\n%s", diff) 101 | } 102 | } 103 | 104 | // intended to be called like: `defer setenvWithUnset(...)()` 105 | func setenvWithUnset(key string, value string) func() { 106 | os.Setenv(key, value) 107 | return func() { os.Unsetenv(key) } 108 | } 109 | -------------------------------------------------------------------------------- /internal/plan/doc.go: -------------------------------------------------------------------------------- 1 | // Package plan provides the model for the test plan. 2 | package plan 3 | -------------------------------------------------------------------------------- /internal/plan/fallback.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "cmp" 5 | "slices" 6 | "strconv" 7 | ) 8 | 9 | // CreateFallbackPlan creates a fallback test plan for the given tests and parallelism. 10 | // It distributes test cases evenly across the tasks using deterministic algorithm. 11 | func CreateFallbackPlan(files []string, parallelism int) TestPlan { 12 | // sort all test cases 13 | slices.SortFunc(files, func(a, b string) int { 14 | return cmp.Compare(a, b) 15 | }) 16 | 17 | tasks := make(map[string]*Task) 18 | for i := 0; i < parallelism; i++ { 19 | tasks[strconv.Itoa(i)] = &Task{ 20 | NodeNumber: i, 21 | Tests: []TestCase{}, 22 | } 23 | } 24 | 25 | // distribute files to tasks 26 | for i, file := range files { 27 | nodeNumber := i % parallelism 28 | task := tasks[strconv.Itoa(nodeNumber)] 29 | task.Tests = append(task.Tests, TestCase{ 30 | Path: file, 31 | }) 32 | } 33 | 34 | return TestPlan{ 35 | Tasks: tasks, 36 | Fallback: true, 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal/plan/fallback_test.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | ) 8 | 9 | func TestCreateFallbackPlan(t *testing.T) { 10 | scenarios := []struct { 11 | files []string 12 | parallelism int 13 | want [][]TestCase 14 | }{ 15 | { 16 | files: []string{"a", "b", "c", "d", "e"}, 17 | parallelism: 2, 18 | want: [][]TestCase{ 19 | {{Path: "a"}, {Path: "c"}, {Path: "e"}}, 20 | {{Path: "b"}, {Path: "d"}}, 21 | }, 22 | }, 23 | { 24 | files: []string{"a", "c", "b", "e", "d"}, 25 | parallelism: 3, 26 | want: [][]TestCase{ 27 | {{Path: "a"}, {Path: "d"}}, 28 | {{Path: "b"}, {Path: "e"}}, 29 | {{Path: "c"}}, 30 | }, 31 | }, 32 | // The function should allow for duplicate test cases in the input 33 | // and distribute them evenly across the nodes. 34 | // This can be useful when the same test case needs to be run multiple times 35 | // to ensure it's not flaky. 36 | // Preventing duplicate test cases will be the responsibility of the caller. 37 | { 38 | files: []string{"a", "a", "b", "c", "d", "c"}, 39 | parallelism: 4, 40 | want: [][]TestCase{ 41 | {{Path: "a"}, {Path: "c"}}, 42 | {{Path: "a"}, {Path: "d"}}, 43 | {{Path: "b"}}, 44 | {{Path: "c"}}, 45 | }, 46 | }, 47 | { 48 | files: []string{"a", "b"}, 49 | parallelism: 3, 50 | want: [][]TestCase{ 51 | {{Path: "a"}}, 52 | {{Path: "b"}}, 53 | {}, 54 | }, 55 | }, 56 | } 57 | 58 | for _, s := range scenarios { 59 | plan := CreateFallbackPlan(s.files, s.parallelism) 60 | got := make([][]TestCase, s.parallelism) 61 | for _, task := range plan.Tasks { 62 | got[task.NodeNumber] = task.Tests 63 | } 64 | 65 | if !plan.Fallback { 66 | t.Errorf("CreateFallbackPlan(%v, %v) Fallback is %v, want %v", s.files, s.parallelism, plan.Fallback, true) 67 | } 68 | 69 | if diff := cmp.Diff(got, s.want); diff != "" { 70 | t.Errorf("CreateFallbackPlan(%v, %v) diff (-got +want):\n%s", s.files, s.parallelism, diff) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /internal/plan/type.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | type TestCaseFormat string 4 | 5 | const ( 6 | TestCaseFormatFile TestCaseFormat = "file" 7 | TestCaseFormatExample TestCaseFormat = "example" 8 | ) 9 | 10 | // TestCase currently can represent a single test case or a single test file (when used as output of test plan API). 11 | // TODO: it's best if we split this into two types. 12 | type TestCase struct { 13 | EstimatedDuration int `json:"estimated_duration,omitempty"` 14 | Format TestCaseFormat `json:"format,omitempty"` 15 | Identifier string `json:"identifier,omitempty"` 16 | Name string `json:"name,omitempty"` 17 | // Path is the path of the individual test or test file that the test runner can interpret. 18 | // For example: 19 | // In RSpec, the path can be a test file like `user_spec.rb` or an individual test id like `user_spec.rb[1,2]`. 20 | // In Jest, the path is a test file like `src/components/Button.spec.tsx`. 21 | // In pytest, the path can be a test file like `test_hello.py` or a node id like `test_hello.py::TestHello::test_greet` 22 | // In go test, the path can only be package name like "example.com/foo/bar". 23 | Path string `json:"path"` 24 | Scope string `json:"scope,omitempty"` 25 | } 26 | 27 | // Task represents the task for the given node. 28 | type Task struct { 29 | NodeNumber int `json:"node_number"` 30 | // When splitting by file, this tests array is essentially an array of test files. 31 | // When splitting by example, this array is an array of proper test cases. 32 | // See comment above, we plan to split TestCase into two types or clarify its usage. 33 | Tests []TestCase `json:"tests"` 34 | } 35 | 36 | // TestPlan represents the entire test plan. 37 | type TestPlan struct { 38 | Experiment string `json:"experiment"` 39 | Tasks map[string]*Task `json:"tasks"` 40 | Fallback bool 41 | MutedTests []TestCase `json:"muted_tests,omitempty"` 42 | SkippedTests []TestCase `json:"skipped_tests,omitempty"` 43 | } 44 | -------------------------------------------------------------------------------- /internal/runner/command.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "os/signal" 8 | "strings" 9 | "syscall" 10 | ) 11 | 12 | // runAndForwardSignal runs the command and forwards any signals received to the command. 13 | func runAndForwardSignal(cmd *exec.Cmd) error { 14 | cmd.Stderr = os.Stderr 15 | cmd.Stdout = os.Stdout 16 | 17 | // Create a channel that will be closed when the command finishes. 18 | finishCh := make(chan struct{}) 19 | defer close(finishCh) 20 | 21 | fmt.Println(strings.Join(cmd.Args, " ")) 22 | fmt.Println("") 23 | 24 | if err := cmd.Start(); err != nil { 25 | return err 26 | } 27 | 28 | // Start a goroutine that waits for a signal or the command to finish. 29 | go func() { 30 | // Create another channel to receive the signals. 31 | sigCh := make(chan os.Signal, 1) 32 | signal.Notify(sigCh) 33 | 34 | // Wait for a signal to be received or the command to finish. 35 | // Because a message can come through both channels asynchronously, 36 | // we use for loop to listen to both channels and select the one that has a message. 37 | // Without for loop, only one case would be selected and the other would be ignored. 38 | // If the signal is received first, the finishCh will never get processed and the goroutine will run forever. 39 | for { 40 | select { 41 | case sig := <-sigCh: 42 | // Check if the signal should be ignored (e.g., SIGCHLD on Unix). 43 | if isIgnoredSignal(sig) { 44 | continue 45 | } 46 | // Ignore the error when sending the signal to the command. 47 | _ = cmd.Process.Signal(sig) 48 | case <-finishCh: 49 | // When the the command finishes, we stop listening for signals and return. 50 | signal.Stop(sigCh) 51 | return 52 | } 53 | } 54 | }() 55 | 56 | // Wait for the command to finish. 57 | err := cmd.Wait() 58 | 59 | if err != nil { 60 | // If the command was signaled, return a ProcessProcessSignaledError. 61 | if exitError, ok := err.(*exec.ExitError); ok { 62 | if status, ok := exitError.Sys().(syscall.WaitStatus); ok && status.Signaled() { 63 | return &ProcessSignaledError{Signal: status.Signal()} 64 | } 65 | } 66 | return err 67 | } 68 | 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /internal/runner/command_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "os/exec" 7 | "syscall" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestRunAndForwardSignal(t *testing.T) { 13 | cmd := exec.Command("echo", "hello world") 14 | 15 | err := runAndForwardSignal(cmd) 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | } 20 | 21 | func TestRunAndForwardSignal_CommandExitsWithNonZero(t *testing.T) { 22 | cmd := exec.Command("false") 23 | 24 | err := runAndForwardSignal(cmd) 25 | exitError := new(exec.ExitError) 26 | if !errors.As(err, &exitError) { 27 | t.Fatalf("runAndForwardSignal(cmd) error type = %T (%v), want *exec.ExitError", err, err) 28 | } 29 | if exitError.ExitCode() != 1 { 30 | t.Errorf("exitError.ExitCode() = %d, want 1", exitError.ExitCode()) 31 | } 32 | } 33 | 34 | func TestRunAndForwardSignal_SignalReceivedInMainProcess(t *testing.T) { 35 | cmd := exec.Command("sleep", "10") 36 | 37 | // Send a SIGTERM signal to the main process. 38 | go func() { 39 | pid := os.Getpid() 40 | process, err := os.FindProcess(pid) 41 | if err != nil { 42 | t.Errorf("os.FindProcess(%d) error = %v", pid, err) 43 | } 44 | time.Sleep(300 * time.Millisecond) 45 | process.Signal(syscall.SIGTERM) 46 | }() 47 | 48 | err := runAndForwardSignal(cmd) 49 | 50 | signalError := new(ProcessSignaledError) 51 | if !errors.As(err, &signalError) { 52 | t.Errorf("runAndForwardSignal(cmd) error type = %T (%v), want *ErrProcessSignaled", err, err) 53 | } 54 | if signalError.Signal != syscall.SIGTERM { 55 | t.Errorf("runAndForwardSignal(cmd) signal = %d, want %d", syscall.SIGTERM, signalError.Signal) 56 | } 57 | } 58 | 59 | func TestRunAndForwardSignal_SignalReceivedInSubProcess(t *testing.T) { 60 | cmd := exec.Command("./testdata/segv.sh") 61 | 62 | err := runAndForwardSignal(cmd) 63 | 64 | signalError := new(ProcessSignaledError) 65 | if !errors.As(err, &signalError) { 66 | t.Errorf("runAndForwardSignal(cmd) error type = %T (%v), want *ErrProcessSignaled", err, err) 67 | } 68 | if signalError.Signal != syscall.SIGSEGV { 69 | t.Errorf("runAndForwardSignal(cmd) signal = %d, want %d", syscall.SIGSEGV, signalError.Signal) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/runner/cypress.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | "slices" 7 | "strings" 8 | 9 | "github.com/buildkite/test-engine-client/internal/debug" 10 | "github.com/buildkite/test-engine-client/internal/plan" 11 | "github.com/kballard/go-shellquote" 12 | ) 13 | 14 | type Cypress struct { 15 | RunnerConfig 16 | } 17 | 18 | func (c Cypress) Name() string { 19 | return "Cypress" 20 | } 21 | 22 | func NewCypress(c RunnerConfig) Cypress { 23 | if c.TestCommand == "" { 24 | c.TestCommand = "npx cypress run --spec {{testExamples}}" 25 | } 26 | 27 | if c.TestFilePattern == "" { 28 | c.TestFilePattern = "**/*.cy.{js,jsx,ts,tsx}" 29 | } 30 | 31 | return Cypress{ 32 | RunnerConfig: c, 33 | } 34 | } 35 | 36 | func (c Cypress) Run(result *RunResult, testCases []plan.TestCase, retry bool) error { 37 | testPaths := make([]string, len(testCases)) 38 | for i, tc := range testCases { 39 | testPaths[i] = tc.Path 40 | } 41 | cmdName, cmdArgs, err := c.commandNameAndArgs(c.TestCommand, testPaths) 42 | if err != nil { 43 | return fmt.Errorf("failed to build command: %w", err) 44 | } 45 | 46 | cmd := exec.Command(cmdName, cmdArgs...) 47 | 48 | err = runAndForwardSignal(cmd) 49 | 50 | return err 51 | } 52 | 53 | func (c Cypress) GetFiles() ([]string, error) { 54 | debug.Println("Discovering test files with include pattern:", c.TestFilePattern, "exclude pattern:", c.TestFileExcludePattern) 55 | files, err := discoverTestFiles(c.TestFilePattern, c.TestFileExcludePattern) 56 | debug.Println("Discovered", len(files), "files") 57 | 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | if len(files) == 0 { 63 | return nil, fmt.Errorf("no files found with pattern %q and exclude pattern %q", c.TestFilePattern, c.TestFileExcludePattern) 64 | } 65 | 66 | return files, nil 67 | } 68 | 69 | func (c Cypress) GetExamples(files []string) ([]plan.TestCase, error) { 70 | return nil, fmt.Errorf("not supported in Cypress") 71 | } 72 | 73 | func (c Cypress) commandNameAndArgs(cmd string, testCases []string) (string, []string, error) { 74 | words, err := shellquote.Split(cmd) 75 | if err != nil { 76 | return "", []string{}, err 77 | } 78 | idx := slices.Index(words, "{{testExamples}}") 79 | specs := strings.Join(testCases, ",") 80 | if idx < 0 { 81 | words = append(words, "--spec", specs) 82 | } else { 83 | words[idx] = specs 84 | } 85 | 86 | return words[0], words[1:], nil 87 | } 88 | -------------------------------------------------------------------------------- /internal/runner/cypress_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "errors" 5 | "os/exec" 6 | "syscall" 7 | "testing" 8 | 9 | "github.com/buildkite/test-engine-client/internal/plan" 10 | "github.com/google/go-cmp/cmp" 11 | "github.com/kballard/go-shellquote" 12 | ) 13 | 14 | func TestCypressRun(t *testing.T) { 15 | changeCwd(t, "./testdata/cypress") 16 | 17 | cypress := NewCypress(RunnerConfig{ 18 | TestCommand: "yarn cypress run --spec {{testExamples}}", 19 | }) 20 | 21 | testCases := []plan.TestCase{ 22 | {Path: "./cypress/e2e/passing_spec.cy.js"}, 23 | } 24 | result := NewRunResult([]plan.TestCase{}) 25 | err := cypress.Run(result, testCases, false) 26 | 27 | if err != nil { 28 | t.Errorf("Cypress.Run(%q) error = %v", testCases, err) 29 | } 30 | 31 | if result.Status() != RunStatusUnknown { 32 | t.Errorf("Cypress.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusUnknown) 33 | } 34 | } 35 | 36 | func TestCypressRun_TestFailed(t *testing.T) { 37 | changeCwd(t, "./testdata/cypress") 38 | 39 | cypress := NewCypress(RunnerConfig{ 40 | TestCommand: "yarn cypress run --spec {{testExamples}}", 41 | }) 42 | 43 | testCases := []plan.TestCase{ 44 | {Path: "./cypress/e2e/failing_spec.cy.js"}, 45 | {Path: "./cypress/e2e/passing_spec.cy.js"}, 46 | } 47 | result := NewRunResult([]plan.TestCase{}) 48 | err := cypress.Run(result, testCases, false) 49 | 50 | if result.Status() != RunStatusUnknown { 51 | t.Errorf("Cypress.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusUnknown) 52 | } 53 | 54 | exitError := new(exec.ExitError) 55 | if !errors.As(err, &exitError) { 56 | t.Errorf("Cypress.Run(%q) error type = %T (%v), want *exec.ExitError", testCases, err, err) 57 | } 58 | } 59 | 60 | func TestCypressRun_CommandFailed(t *testing.T) { 61 | cypress := NewCypress(RunnerConfig{ 62 | TestCommand: "yarn cypress run --json", 63 | }) 64 | 65 | testCases := []plan.TestCase{} 66 | result := NewRunResult([]plan.TestCase{}) 67 | err := cypress.Run(result, testCases, false) 68 | 69 | if result.Status() != RunStatusUnknown { 70 | t.Errorf("Cypress.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusUnknown) 71 | } 72 | 73 | exitError := new(exec.ExitError) 74 | if !errors.As(err, &exitError) { 75 | t.Errorf("Cypress.Run(%q) error type = %T (%v), want *exec.ExitError", testCases, err, err) 76 | } 77 | } 78 | 79 | func TestCypressRun_SignaledError(t *testing.T) { 80 | cypress := NewCypress(RunnerConfig{ 81 | TestCommand: "./testdata/segv.sh", 82 | }) 83 | 84 | testCases := []plan.TestCase{ 85 | {Path: "./doesnt-matter.cy.js"}, 86 | } 87 | result := NewRunResult([]plan.TestCase{}) 88 | err := cypress.Run(result, testCases, false) 89 | 90 | if result.Status() != RunStatusUnknown { 91 | t.Errorf("Cypress.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusUnknown) 92 | } 93 | 94 | signalError := new(ProcessSignaledError) 95 | if !errors.As(err, &signalError) { 96 | t.Errorf("Cypress.Run(%q) error type = %T (%v), want *ErrProcessSignaled", testCases, err, err) 97 | } 98 | if signalError.Signal != syscall.SIGSEGV { 99 | t.Errorf("Cypress.Run(%q) signal = %d, want %d", testCases, syscall.SIGSEGV, signalError.Signal) 100 | } 101 | } 102 | 103 | func TestCypressGetFiles(t *testing.T) { 104 | cypress := NewCypress(RunnerConfig{}) 105 | 106 | got, err := cypress.GetFiles() 107 | if err != nil { 108 | t.Errorf("Cypress.GetFiles() error = %v", err) 109 | } 110 | 111 | want := []string{ 112 | "testdata/cypress/cypress/e2e/failing_spec.cy.js", 113 | "testdata/cypress/cypress/e2e/flaky_spec.cy.js", 114 | "testdata/cypress/cypress/e2e/passing_spec.cy.js", 115 | } 116 | 117 | if diff := cmp.Diff(got, want); diff != "" { 118 | t.Errorf("Cypress.GetFiles() diff (-got +want):\n%s", diff) 119 | } 120 | } 121 | 122 | func TestCypressCommandNameAndArgs_WithInterpolationPlaceholder(t *testing.T) { 123 | testCases := []string{"cypress/e2e/passing_spec.cy.js", "cypress/e2e/flaky_spec.cy.js"} 124 | testCommand := "cypress run --spec {{testExamples}}" 125 | 126 | cy := NewCypress(RunnerConfig{ 127 | TestCommand: testCommand, 128 | ResultPath: "cypress.json", 129 | }) 130 | 131 | gotName, gotArgs, err := cy.commandNameAndArgs(testCommand, testCases) 132 | if err != nil { 133 | t.Errorf("commandNameAndArgs(%q, %q) error = %v", testCases, testCommand, err) 134 | } 135 | 136 | wantName := "cypress" 137 | wantArgs := []string{"run", "--spec", "cypress/e2e/passing_spec.cy.js,cypress/e2e/flaky_spec.cy.js"} 138 | 139 | if diff := cmp.Diff(gotName, wantName); diff != "" { 140 | t.Errorf("commandNameAndArgs(%q, %q) diff (-got +want):\n%s", testCases, testCommand, diff) 141 | } 142 | if diff := cmp.Diff(gotArgs, wantArgs); diff != "" { 143 | t.Errorf("commandNameAndArgs(%q, %q) diff (-got +want):\n%s", testCases, testCommand, diff) 144 | } 145 | } 146 | 147 | func TestCypressCommandNameAndArgs_WithoutTestExamplesPlaceholder(t *testing.T) { 148 | testCases := []string{"cypress/e2e/passing_spec.cy.js", "cypress/e2e/flaky_spec.cy.js"} 149 | testCommand := "cypress run" 150 | 151 | cypress := NewCypress(RunnerConfig{ 152 | TestCommand: testCommand, 153 | }) 154 | 155 | gotName, gotArgs, err := cypress.commandNameAndArgs(testCommand, testCases) 156 | if err != nil { 157 | t.Errorf("commandNameAndArgs(%q, %q) error = %v", testCases, testCommand, err) 158 | } 159 | 160 | wantName := "cypress" 161 | wantArgs := []string{"run", "--spec", "cypress/e2e/passing_spec.cy.js,cypress/e2e/flaky_spec.cy.js"} 162 | 163 | if diff := cmp.Diff(gotName, wantName); diff != "" { 164 | t.Errorf("commandNameAndArgs(%q, %q) diff (-got +want):\n%s", testCases, testCommand, diff) 165 | } 166 | if diff := cmp.Diff(gotArgs, wantArgs); diff != "" { 167 | t.Errorf("commandNameAndArgs(%q, %q) diff (-got +want):\n%s", testCases, testCommand, diff) 168 | } 169 | } 170 | 171 | func TestCypressCommandNameAndArgs_InvalidTestCommand(t *testing.T) { 172 | testCases := []string{"cypress/e2e/passing_spec.cy.js", "cypress/e2e/flaky_spec.cy.js"} 173 | testCommand := "cypress run --options '{{testExamples}}" 174 | 175 | cypress := NewCypress(RunnerConfig{ 176 | TestCommand: testCommand, 177 | }) 178 | 179 | gotName, gotArgs, err := cypress.commandNameAndArgs(testCommand, testCases) 180 | 181 | wantName := "" 182 | wantArgs := []string{} 183 | 184 | if diff := cmp.Diff(gotName, wantName); diff != "" { 185 | t.Errorf("commandNameAndArgs() diff (-got +want):\n%s", diff) 186 | } 187 | if diff := cmp.Diff(gotArgs, wantArgs); diff != "" { 188 | t.Errorf("commandNameAndArgs() diff (-got +want):\n%s", diff) 189 | } 190 | if !errors.Is(err, shellquote.UnterminatedSingleQuoteError) { 191 | t.Errorf("commandNameAndArgs() error = %v, want %v", err, shellquote.UnterminatedSingleQuoteError) 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /internal/runner/detector.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/buildkite/test-engine-client/internal/config" 7 | "github.com/buildkite/test-engine-client/internal/plan" 8 | ) 9 | 10 | type RunnerConfig struct { 11 | TestRunner string 12 | TestCommand string 13 | TestFilePattern string 14 | TestFileExcludePattern string 15 | RetryTestCommand string 16 | // ResultPath is used internally so bktec can read result from Test Runner. 17 | // User typically don't need to worry about setting this except in in RSpec and playwright. 18 | // In playwright, for example, it can only be configured via a config file, therefore it's mandatory for user to set. 19 | ResultPath string 20 | } 21 | 22 | type TestRunner interface { 23 | Run(result *RunResult, testCases []plan.TestCase, retry bool) error 24 | GetExamples(files []string) ([]plan.TestCase, error) 25 | GetFiles() ([]string, error) 26 | Name() string 27 | } 28 | 29 | func DetectRunner(cfg config.Config) (TestRunner, error) { 30 | var runnerConfig = RunnerConfig{ 31 | TestRunner: cfg.TestRunner, 32 | TestCommand: cfg.TestCommand, 33 | TestFilePattern: cfg.TestFilePattern, 34 | TestFileExcludePattern: cfg.TestFileExcludePattern, 35 | RetryTestCommand: cfg.RetryCommand, 36 | ResultPath: cfg.ResultPath, 37 | } 38 | 39 | switch cfg.TestRunner { 40 | case "rspec": 41 | return NewRspec(runnerConfig), nil 42 | case "jest": 43 | return NewJest(runnerConfig), nil 44 | case "cypress": 45 | return NewCypress(runnerConfig), nil 46 | case "playwright": 47 | return NewPlaywright(runnerConfig), nil 48 | case "pytest": 49 | return NewPytest(runnerConfig), nil 50 | case "pytest-pants": 51 | return NewPytestPants(runnerConfig), nil 52 | case "gotest": 53 | return NewGoTest(runnerConfig), nil 54 | default: 55 | // Update the error message to include the new runner 56 | return nil, errors.New("runner value is invalid, possible values are 'rspec', 'jest', 'cypress', 'playwright', 'pytest', 'pytest-pants', or 'gotest'") 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /internal/runner/discover.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "fmt" 5 | "io/fs" 6 | 7 | "drjosh.dev/zzglob" 8 | ) 9 | 10 | func discoverTestFiles(pattern string, excludePattern string) ([]string, error) { 11 | parsedPattern, err := zzglob.Parse(pattern) 12 | if err != nil { 13 | return nil, fmt.Errorf("error parsing test file pattern %q", pattern) 14 | } 15 | 16 | parsedExcludePattern, err := zzglob.Parse(excludePattern) 17 | if err != nil { 18 | return nil, fmt.Errorf("error parsing test file exclude pattern %q", excludePattern) 19 | } 20 | 21 | discoveredFiles := []string{} 22 | 23 | // Use the Glob function to traverse the directory recursively 24 | // and append the matched file paths to the discoveredFiles slice 25 | err = parsedPattern.Glob(func(path string, d fs.DirEntry, err error) error { 26 | if err != nil { 27 | fmt.Printf("Error walking at path %q: %v\n", path, err) 28 | return nil 29 | } 30 | 31 | // Check if the path matches the exclude pattern. If so, skip it. 32 | // If it matches a directory, then skip that directory. 33 | if parsedExcludePattern.Match(path) { 34 | if d.IsDir() { 35 | return fs.SkipDir 36 | } 37 | return nil 38 | } 39 | 40 | // Skip the node_modules directory 41 | if d.Name() == "node_modules" { 42 | return fs.SkipDir 43 | } 44 | 45 | // Skip directories that happen to match the include pattern - we're 46 | // only interested in files. 47 | if d.IsDir() { 48 | return nil 49 | } 50 | 51 | discoveredFiles = append(discoveredFiles, path) 52 | return nil 53 | }, zzglob.WalkIntermediateDirs(true)) 54 | 55 | if err != nil { 56 | return nil, fmt.Errorf("error walking directory: %v", err) 57 | } 58 | 59 | return discoveredFiles, nil 60 | } 61 | -------------------------------------------------------------------------------- /internal/runner/discover_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | ) 8 | 9 | func TestDiscoverTestFiles(t *testing.T) { 10 | pattern := "testdata/files/**/*_test" 11 | got, err := discoverTestFiles(pattern, "") 12 | 13 | if err != nil { 14 | t.Errorf("discoverTestFiles(%q, %q) error: %v", pattern, "", err) 15 | } 16 | 17 | want := []string{ 18 | "testdata/files/animals/ant_test", 19 | "testdata/files/animals/bee_test", 20 | "testdata/files/fruits/apple_test", 21 | "testdata/files/fruits/banana_test", 22 | "testdata/files/vegetable_test", 23 | } 24 | 25 | if diff := cmp.Diff(got, want); diff != "" { 26 | t.Errorf("discoverTestFiles(%q, %q) diff (-got +want):\n%s", pattern, "", diff) 27 | } 28 | } 29 | 30 | func TestDiscoverTestFiles_WithExcludePattern(t *testing.T) { 31 | pattern := "testdata/files/**/*_test" 32 | excludePattern := "testdata/files/**/animals/*" 33 | got, err := discoverTestFiles(pattern, excludePattern) 34 | 35 | if err != nil { 36 | t.Errorf("discoverTestFiles(%q, %q) error: %v", pattern, excludePattern, err) 37 | } 38 | 39 | want := []string{ 40 | "testdata/files/fruits/apple_test", 41 | "testdata/files/fruits/banana_test", 42 | "testdata/files/vegetable_test", 43 | } 44 | 45 | if diff := cmp.Diff(got, want); diff != "" { 46 | t.Errorf("discoverTestFiles(%q, %q) diff (-got +want):\n%s", pattern, excludePattern, diff) 47 | } 48 | } 49 | 50 | func TestDiscoverTestFiles_WithExcludeDirectory(t *testing.T) { 51 | pattern := "testdata/files/**/*_test" 52 | excludePattern := "testdata/files/**/animals" 53 | got, err := discoverTestFiles(pattern, excludePattern) 54 | 55 | if err != nil { 56 | t.Errorf("discoverTestFiles(%q, %q) error: %v", pattern, excludePattern, err) 57 | } 58 | 59 | want := []string{ 60 | "testdata/files/fruits/apple_test", 61 | "testdata/files/fruits/banana_test", 62 | "testdata/files/vegetable_test", 63 | } 64 | 65 | if diff := cmp.Diff(got, want); diff != "" { 66 | t.Errorf("discoverTestFiles(%q, %q) diff (-got +want):\n%s", pattern, excludePattern, diff) 67 | } 68 | } 69 | 70 | func TestDiscoverTestFiles_ExcludeNodeModules(t *testing.T) { 71 | pattern := "testdata/**/*.js" 72 | excludePattern := "" 73 | got, err := discoverTestFiles(pattern, excludePattern) 74 | 75 | if err != nil { 76 | t.Errorf("discoverTestFiles(%q, %q) error: %v", pattern, excludePattern, err) 77 | } 78 | 79 | want := []string{ 80 | "testdata/cypress/cypress/e2e/failing_spec.cy.js", 81 | "testdata/cypress/cypress/e2e/flaky_spec.cy.js", 82 | "testdata/cypress/cypress/e2e/passing_spec.cy.js", 83 | "testdata/cypress/cypress.config.js", 84 | "testdata/jest/failure.spec.js", 85 | "testdata/jest/jest.config.js", 86 | "testdata/jest/runtimeError.spec.js", 87 | "testdata/jest/skipped.spec.js", 88 | "testdata/jest/slow.spec.js", 89 | "testdata/jest/spells/expelliarmus.spec.js", 90 | "testdata/playwright/playwright.config.js", 91 | "testdata/playwright/tests/error.spec.js", 92 | "testdata/playwright/tests/example.spec.js", 93 | "testdata/playwright/tests/failed.spec.js", 94 | "testdata/playwright/tests/skipped.spec.js", 95 | } 96 | 97 | if diff := cmp.Diff(got, want); diff != "" { 98 | t.Errorf("discoverTestFiles(%q, %q) diff (-got +want):\n%s", pattern, excludePattern, diff) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /internal/runner/doc.go: -------------------------------------------------------------------------------- 1 | // Package runner provides the test runners that run sets of test cases. 2 | package runner 3 | -------------------------------------------------------------------------------- /internal/runner/error.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "fmt" 5 | "syscall" 6 | ) 7 | 8 | type ProcessSignaledError struct { 9 | Signal syscall.Signal 10 | } 11 | 12 | func (e *ProcessSignaledError) Error() string { 13 | return fmt.Sprintf("process was signaled with signal %d", e.Signal) 14 | } 15 | -------------------------------------------------------------------------------- /internal/runner/gotest.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os/exec" 7 | "strings" 8 | 9 | "github.com/buildkite/test-engine-client/internal/debug" 10 | "github.com/buildkite/test-engine-client/internal/plan" 11 | "github.com/kballard/go-shellquote" 12 | ) 13 | 14 | type GoTest struct { 15 | RunnerConfig 16 | } 17 | 18 | // Compile-time check that GoTest implements TestRunner 19 | var _ TestRunner = (*GoTest)(nil) 20 | 21 | func NewGoTest(c RunnerConfig) GoTest { 22 | if c.TestCommand == "" { 23 | c.TestCommand = "gotestsum --junitfile={{resultPath}} {{packages}}" 24 | } 25 | 26 | if c.RetryTestCommand == "" { 27 | c.RetryTestCommand = c.TestCommand 28 | } 29 | 30 | return GoTest{ 31 | RunnerConfig: c, 32 | } 33 | } 34 | 35 | func (g GoTest) Name() string { 36 | return "gotest" 37 | } 38 | 39 | func (g GoTest) GetExamples(files []string) ([]plan.TestCase, error) { 40 | return nil, fmt.Errorf("not supported in go test") 41 | } 42 | 43 | // Run executes the configured command for the specified packages. 44 | func (g GoTest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error { 45 | cmdName, cmdArgs, err := g.commandNameAndArgs(g.TestCommand, testCases) 46 | if err != nil { 47 | return fmt.Errorf("failed to build command: %w", err) 48 | } 49 | 50 | cmd := exec.Command(cmdName, cmdArgs...) 51 | err = runAndForwardSignal(cmd) 52 | // go test output does not differentiate build fail or test fail. They both return 1 53 | // What is even more bizarre is that even when go test failed on compliation, it will still generate an output xml 54 | // file that says "TestMain" failed.. 55 | if exitError := new(exec.ExitError); errors.As(err, &exitError) && exitError.ExitCode() != 1 { 56 | return err 57 | } 58 | 59 | testResults, err := loadAndParseGotestJUnitXmlResult(g.ResultPath) 60 | 61 | if err != nil { 62 | return fmt.Errorf("failed to load and parse test result: %w", err) 63 | } 64 | 65 | for _, test := range testResults { 66 | result.RecordTestResult(plan.TestCase{ 67 | Format: plan.TestCaseFormatExample, 68 | Scope: test.Classname, 69 | Name: test.Name, 70 | // This is the special thing about go test support. 71 | Path: test.Classname, 72 | }, test.Result) 73 | } 74 | 75 | return nil // Success 76 | } 77 | 78 | // GetFiles discovers Go packages using `go list ./...`. 79 | // Note that "file" does not exist as a first level concept in Golang projects 80 | // So this func is returning a list of packages instead of files. 81 | // The implication is that the Server-side smart test splitting will never work. 82 | // It almost will always fallback to simple splitting. 83 | func (g GoTest) GetFiles() ([]string, error) { 84 | debug.Println("Discovering Go packages with `go list ./...`") 85 | cmd := exec.Command("go", "list", "./...") 86 | output, err := cmd.Output() 87 | if err != nil { 88 | // Handle stderr for better error messages 89 | if ee, ok := err.(*exec.ExitError); ok { 90 | return nil, fmt.Errorf("go list failed: %w\nstderr:\n%s", err, string(ee.Stderr)) 91 | } 92 | return nil, fmt.Errorf("failed to run go list: %w", err) 93 | } 94 | packages := strings.Split(strings.TrimSpace(string(output)), "\n") 95 | // Filter out empty strings if any 96 | validPackages := []string{} 97 | for _, pkg := range packages { 98 | if pkg != "" { 99 | validPackages = append(validPackages, pkg) 100 | } 101 | } 102 | debug.Println("Discovered", len(validPackages), "packages") 103 | if len(validPackages) == 0 { 104 | return nil, fmt.Errorf("no Go packages found using `go list ./...`") 105 | } 106 | return validPackages, nil 107 | } 108 | 109 | func (p GoTest) commandNameAndArgs(cmd string, testCases []plan.TestCase) (string, []string, error) { 110 | packages, err := p.getPackages(testCases) 111 | if err != nil { 112 | return "", []string{}, nil 113 | } 114 | 115 | concatenatedPackages := strings.Join(packages, " ") 116 | 117 | if strings.Contains(cmd, "{{packages}}") { 118 | cmd = strings.Replace(cmd, "{{packages}}", concatenatedPackages, 1) 119 | } else { 120 | cmd = cmd + " " + concatenatedPackages 121 | } 122 | 123 | cmd = strings.Replace(cmd, "{{resultPath}}", p.ResultPath, 1) 124 | 125 | args, err := shellquote.Split(cmd) 126 | 127 | if err != nil { 128 | return "", []string{}, err 129 | } 130 | 131 | return args[0], args[1:], nil 132 | } 133 | 134 | // Pluck unique packages from test cases 135 | func (g GoTest) getPackages(testCases []plan.TestCase) ([]string, error) { 136 | packages := make([]string, 0, len(testCases)) 137 | 138 | packagesSeen := map[string]bool{} 139 | for _, tc := range testCases { 140 | packageName := tc.Path 141 | if !packagesSeen[packageName] { 142 | packages = append(packages, packageName) 143 | packagesSeen[packageName] = true 144 | } 145 | } 146 | if len(packages) == 0 { 147 | // The likelihood of this is very low 148 | return nil, fmt.Errorf("Unable to extract package names from test plan") 149 | } 150 | debug.Printf("Packages: %v\n", packages) 151 | 152 | return packages, nil 153 | } 154 | -------------------------------------------------------------------------------- /internal/runner/gotest_junit.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "encoding/xml" 5 | "fmt" 6 | "io" 7 | "os" 8 | ) 9 | 10 | // Struct to decode gotestsun --junitfile=... 11 | type GoTestJUnitResult struct { 12 | Classname string `xml:"classname,attr"` 13 | Name string `xml:"name,attr"` 14 | Result TestStatus // passed | failed | skipped 15 | Failure *JUnitXMLFailure `xml:"failure"` 16 | Skipped *JUnitXMLSkipped `xml:"skipped"` 17 | } 18 | 19 | // JUnitXMLFailure represents the element in JUnit XML 20 | type JUnitXMLFailure struct { 21 | Message string `xml:"message,attr"` 22 | Type string `xml:"type,attr"` 23 | Content string `xml:",chardata"` 24 | } 25 | 26 | // JUnitXMLSkipped represents the element in JUnit XML 27 | type JUnitXMLSkipped struct { 28 | Message string `xml:"message,attr"` 29 | } 30 | 31 | // JUnitXMLTestSuite represents the element in JUnit XML 32 | type JUnitXMLTestSuite struct { 33 | XMLName xml.Name `xml:"testsuite"` 34 | Name string `xml:"name,attr"` 35 | Tests int `xml:"tests,attr"` 36 | Failures int `xml:"failures,attr"` 37 | Errors int `xml:"errors,attr"` 38 | Time float64 `xml:"time,attr"` 39 | Timestamp string `xml:"timestamp,attr"` 40 | TestCases []GoTestJUnitResult `xml:"testcase"` 41 | } 42 | 43 | // JUnitXMLTestSuites represents the root element in JUnit XML 44 | type JUnitXMLTestSuites struct { 45 | XMLName xml.Name `xml:"testsuites"` 46 | Tests int `xml:"tests,attr"` 47 | Failures int `xml:"failures,attr"` 48 | Errors int `xml:"errors,attr"` 49 | Time float64 `xml:"time,attr"` 50 | TestSuites []JUnitXMLTestSuite `xml:"testsuite"` 51 | } 52 | 53 | func loadAndParseGotestJUnitXmlResult(path string) ([]GoTestJUnitResult, error) { 54 | xmlFile, err := os.Open(path) 55 | if err != nil { 56 | return nil, fmt.Errorf("failed to open JUnit XML file %s: %w", path, err) 57 | } 58 | defer xmlFile.Close() 59 | 60 | byteValue, err := io.ReadAll(xmlFile) 61 | if err != nil { 62 | return nil, fmt.Errorf("failed to read JUnit XML file %s: %w", path, err) 63 | } 64 | 65 | var testSuites JUnitXMLTestSuites 66 | err = xml.Unmarshal(byteValue, &testSuites) 67 | if err != nil { 68 | return nil, fmt.Errorf("failed to unmarshal JUnit XML file %s: %w", path, err) 69 | } 70 | 71 | var results []GoTestJUnitResult 72 | for _, suite := range testSuites.TestSuites { 73 | for _, tc := range suite.TestCases { 74 | testCase := tc // Create a new variable to avoid closure capturing the loop variable 75 | if testCase.Failure != nil { 76 | testCase.Result = TestStatusFailed 77 | } else if testCase.Skipped != nil { 78 | testCase.Result = TestStatusSkipped 79 | } else { 80 | testCase.Result = TestStatusPassed 81 | } 82 | results = append(results, testCase) 83 | } 84 | } 85 | 86 | return results, nil 87 | } 88 | -------------------------------------------------------------------------------- /internal/runner/gotest_junit_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | const exampleJUnitXML = ` 12 | 13 | 14 | 15 | 16 | 17 | 18 | === RUN TestPrintf debug_test.go:21: error matching output: <nil> --- FAIL: TestPrintf (0.00s) 19 | 20 | 21 | 22 | 23 | 24 | ` 25 | 26 | func TestLoadAndParseGotestJUnitXmlResult(t *testing.T) { 27 | tmpfile, err := os.CreateTemp("", "junit.*.xml") 28 | require.NoError(t, err) 29 | defer os.Remove(tmpfile.Name()) // clean up 30 | 31 | _, err = tmpfile.WriteString(exampleJUnitXML) 32 | require.NoError(t, err) 33 | err = tmpfile.Close() 34 | require.NoError(t, err) 35 | 36 | results, err := loadAndParseGotestJUnitXmlResult(tmpfile.Name()) 37 | require.NoError(t, err) 38 | 39 | require.Len(t, results, 4) 40 | 41 | assert.Equal(t, "github.com/buildkite/test-engine-client/internal/debug", results[0].Classname) 42 | assert.Equal(t, "TestPrintf", results[0].Name) 43 | assert.Equal(t, TestStatusFailed, results[0].Result) 44 | assert.NotNil(t, results[0].Failure) 45 | assert.Nil(t, results[0].Skipped) 46 | 47 | assert.Equal(t, "github.com/buildkite/test-engine-client/internal/debug", results[1].Classname) 48 | assert.Equal(t, "TestPrintf_disabled", results[1].Name) 49 | assert.Equal(t, TestStatusPassed, results[1].Result) 50 | assert.Nil(t, results[1].Failure) 51 | assert.Nil(t, results[1].Skipped) 52 | 53 | assert.Equal(t, "github.com/buildkite/test-engine-client/internal/debug", results[2].Classname) 54 | assert.Equal(t, "TestPrintln", results[2].Name) 55 | assert.Equal(t, TestStatusPassed, results[2].Result) 56 | assert.Nil(t, results[2].Failure) 57 | assert.Nil(t, results[2].Skipped) 58 | 59 | assert.Equal(t, "github.com/buildkite/test-engine-client/internal/debug", results[3].Classname) 60 | assert.Equal(t, "TestPrintln_disabled", results[3].Name) 61 | assert.Equal(t, TestStatusPassed, results[3].Result) 62 | assert.Nil(t, results[3].Failure) 63 | assert.Nil(t, results[3].Skipped) 64 | } 65 | -------------------------------------------------------------------------------- /internal/runner/gotest_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/buildkite/test-engine-client/internal/plan" 10 | "github.com/google/go-cmp/cmp" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestGotestRun(t *testing.T) { 15 | changeCwd(t, "./testdata/go") 16 | 17 | gotest := NewGoTest(RunnerConfig{ 18 | ResultPath: getRandomXmlTempFilename(), 19 | }) 20 | testCases := []plan.TestCase{ 21 | {Path: "example.com/hello"}, 22 | } 23 | result := NewRunResult([]plan.TestCase{}) 24 | err := gotest.Run(result, testCases, false) 25 | 26 | assert.NoError(t, err) 27 | if result.Status() != RunStatusPassed { 28 | t.Errorf("Gotest.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusPassed) 29 | } 30 | 31 | fmt.Printf("result.tests: %v\n", result.tests) 32 | 33 | testResult := result.tests["example.com/hello/TestHelloWorld/example.com/hello"] 34 | if testResult.Path != "example.com/hello" { 35 | t.Errorf("TestResult.Path = %v, want %v", testResult.Path, "example.com/hello") 36 | } 37 | } 38 | 39 | func TestGotestRun_TestFailed(t *testing.T) { 40 | changeCwd(t, "./testdata/go") 41 | 42 | gotest := NewGoTest(RunnerConfig{ 43 | ResultPath: getRandomXmlTempFilename(), 44 | }) 45 | testCases := []plan.TestCase{ 46 | {Path: "example.com/hello/bad"}, 47 | } 48 | result := NewRunResult([]plan.TestCase{}) 49 | err := gotest.Run(result, testCases, false) 50 | 51 | assert.NoError(t, err) 52 | if result.Status() != RunStatusFailed { 53 | t.Errorf("Gotest.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusFailed) 54 | } 55 | } 56 | 57 | func TestGotestRun_CommandFailed(t *testing.T) { 58 | changeCwd(t, "./testdata/go") 59 | 60 | gotest := NewGoTest(RunnerConfig{ 61 | TestCommand: "gotestsum --junitfile {{resultPath}} bluhbluh", 62 | ResultPath: getRandomXmlTempFilename(), 63 | }) 64 | testCases := []plan.TestCase{ 65 | {Path: "example.com/hello"}, 66 | } 67 | result := NewRunResult([]plan.TestCase{}) 68 | err := gotest.Run(result, testCases, false) 69 | 70 | assert.NoError(t, err) // sadly we don't have a way to reliably differentiate test fail vs build fail (yet). 71 | if result.Status() != RunStatusFailed { 72 | t.Errorf("Gotest.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusFailed) 73 | } 74 | } 75 | 76 | func TestGotestGetFiles(t *testing.T) { 77 | changeCwd(t, "./testdata/go") 78 | 79 | gotest := NewGoTest(RunnerConfig{}) 80 | 81 | got, err := gotest.GetFiles() 82 | if err != nil { 83 | t.Errorf("Gotest.GetFiles() error = %v", err) 84 | } 85 | 86 | want := []string{ 87 | "example.com/hello", 88 | "example.com/hello/bad", 89 | } 90 | 91 | if diff := cmp.Diff(got, want); diff != "" { 92 | t.Errorf("Gotest.GetFiles() diff (-got +want):\n%s", diff) 93 | } 94 | } 95 | 96 | func getRandomXmlTempFilename() string { 97 | tempDir, err := os.MkdirTemp("", "bktec-*") 98 | if err != nil { 99 | panic(err) 100 | } 101 | return filepath.Join(tempDir, "test-results.xml") 102 | } 103 | -------------------------------------------------------------------------------- /internal/runner/playwright.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "slices" 10 | 11 | "github.com/buildkite/test-engine-client/internal/debug" 12 | "github.com/buildkite/test-engine-client/internal/plan" 13 | "github.com/kballard/go-shellquote" 14 | ) 15 | 16 | type Playwright struct { 17 | RunnerConfig 18 | } 19 | 20 | func (p Playwright) Name() string { 21 | return "Playwright" 22 | } 23 | 24 | func NewPlaywright(p RunnerConfig) Playwright { 25 | if p.TestCommand == "" { 26 | p.TestCommand = "npx playwright test" 27 | } 28 | 29 | if p.TestFilePattern == "" { 30 | p.TestFilePattern = "**/{*.spec,*.test}.{ts,js}" 31 | } 32 | 33 | return Playwright{ 34 | RunnerConfig: p, 35 | } 36 | } 37 | 38 | func (p Playwright) Run(result *RunResult, testCases []plan.TestCase, retry bool) error { 39 | testPaths := make([]string, len(testCases)) 40 | for i, tc := range testCases { 41 | testPaths[i] = tc.Path 42 | } 43 | 44 | cmdName, cmdArgs, err := p.commandNameAndArgs(p.TestCommand, testPaths) 45 | if err != nil { 46 | return fmt.Errorf("failed to build command: %w", err) 47 | } 48 | 49 | cmd := exec.Command(cmdName, cmdArgs...) 50 | 51 | err = runAndForwardSignal(cmd) 52 | 53 | if ProcessSignaledError := new(ProcessSignaledError); errors.As(err, &ProcessSignaledError) { 54 | return err 55 | } 56 | 57 | report, parseErr := p.parseReport(p.ResultPath) 58 | if parseErr != nil { 59 | fmt.Println("Buildkite Test Engine Client: Failed to read Playwright output, tests will not be retried.") 60 | return err 61 | } 62 | 63 | for _, suite := range report.Suites { 64 | testResults := p.getTestResultsFromSuite(suite, suite.Title) 65 | for _, testResult := range testResults { 66 | result.RecordTestResult(testResult.TestCase, testResult.Status) 67 | } 68 | } 69 | 70 | if len(report.Errors) > 0 { 71 | result.error = fmt.Errorf("Playwright failed with errors") 72 | } 73 | 74 | return nil 75 | 76 | } 77 | 78 | // getTestCasesFromSuite recursively traverses the Playwright report suite and returns all test cases. 79 | // Playwright's report format is a tree structure, where each suite can contain multiple specs and sub-suites. 80 | // The function traverses the tree and collects failed test cases from the leaf nodes. 81 | func (p Playwright) getTestResultsFromSuite(suite PlaywrightReportSuite, suiteName string) []TestResult { 82 | var testResults []TestResult 83 | 84 | for _, spec := range suite.Specs { 85 | projectName := spec.Tests[0].ProjectName 86 | var status TestStatus 87 | if !spec.Ok { 88 | status = TestStatusFailed 89 | } else if spec.Tests[0].Status == "skipped" { 90 | status = TestStatusSkipped 91 | } else { 92 | status = TestStatusPassed 93 | } 94 | 95 | testResults = append(testResults, TestResult{ 96 | TestCase: plan.TestCase{ 97 | Name: spec.Title, 98 | Path: fmt.Sprintf("%s:%d", spec.File, spec.Line), 99 | // The scope has to match with the scope generated by Buildkite test collector. 100 | // In Buildkite test collector, the scope is generated using Playwright built-in reporter function, titlePath(). 101 | // titlePath function returns an array of suite's title from the root suite down to the current test, 102 | // which is then joined with a space separator to form the scope. 103 | // For more details, see: 104 | // [Buildkite Test Collector - Playwright implementation](https://github.com/buildkite/test-collector-javascript/blob/42b803a618a15a07edf0169038ef4b5eba88f98d/playwright/reporter.js#L47) 105 | // [Playwright titlePath implementation](https://github.com/microsoft/playwright/blob/523e50088a7f982dd96aacdb260dfbd1189159b1/packages/playwright/src/common/test.ts#L126) 106 | // [Playwright suite structure](https://playwright.dev/docs/api/class-suite) 107 | Scope: fmt.Sprintf(" %s %s %s", projectName, suiteName, spec.Title), 108 | }, 109 | Status: status, 110 | }) 111 | } 112 | 113 | for _, subSuite := range suite.Suites { 114 | testResults = append(testResults, p.getTestResultsFromSuite(subSuite, fmt.Sprintf("%s %s", suiteName, subSuite.Title))...) 115 | } 116 | 117 | return testResults 118 | } 119 | 120 | func (p Playwright) commandNameAndArgs(cmd string, testCases []string) (string, []string, error) { 121 | words, err := shellquote.Split(cmd) 122 | if err != nil { 123 | return "", []string{}, err 124 | } 125 | idx := slices.Index(words, "{{testExamples}}") 126 | if idx < 0 { 127 | words = append(words, testCases...) 128 | } else { 129 | words = slices.Replace(words, idx, idx+1, testCases...) 130 | } 131 | 132 | return words[0], words[1:], nil 133 | } 134 | 135 | func (p Playwright) parseReport(path string) (PlaywrightReport, error) { 136 | var report PlaywrightReport 137 | data, err := os.ReadFile(path) 138 | if err != nil { 139 | return PlaywrightReport{}, fmt.Errorf("failed to read playwright output: %v", err) 140 | } 141 | 142 | if err := json.Unmarshal(data, &report); err != nil { 143 | return PlaywrightReport{}, fmt.Errorf("failed to parse playwright output: %s", err) 144 | } 145 | 146 | return report, nil 147 | } 148 | 149 | func (p Playwright) GetFiles() ([]string, error) { 150 | debug.Println("Discovering test files with include pattern:", p.TestFilePattern, "exclude pattern:", p.TestFileExcludePattern) 151 | files, err := discoverTestFiles(p.TestFilePattern, p.TestFileExcludePattern) 152 | debug.Println("Discovered", len(files), "files") 153 | 154 | if err != nil { 155 | return nil, err 156 | } 157 | 158 | if len(files) == 0 { 159 | return nil, fmt.Errorf("no files found with pattern %q and exclude pattern %q", p.TestFilePattern, p.TestFileExcludePattern) 160 | } 161 | 162 | return files, nil 163 | } 164 | 165 | func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error) { 166 | return nil, fmt.Errorf("not supported in Playwright") 167 | } 168 | 169 | type PlaywrightTest struct { 170 | ProjectName string 171 | Status string 172 | } 173 | 174 | type PlaywrightSpec struct { 175 | File string 176 | Line int 177 | Column int 178 | Id string 179 | Title string 180 | Ok bool 181 | Tests []PlaywrightTest 182 | } 183 | 184 | type PlaywrightReportSuite struct { 185 | Title string 186 | Specs []PlaywrightSpec 187 | Suites []PlaywrightReportSuite 188 | } 189 | 190 | type PlaywrightReport struct { 191 | Suites []PlaywrightReportSuite 192 | Stats struct { 193 | Expected int 194 | Unexpected int 195 | } 196 | Errors []struct { 197 | Message string 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /internal/runner/pytest.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/buildkite/test-engine-client/internal/debug" 13 | "github.com/buildkite/test-engine-client/internal/plan" 14 | "github.com/kballard/go-shellquote" 15 | ) 16 | 17 | type Pytest struct { 18 | RunnerConfig 19 | } 20 | 21 | func (p Pytest) Name() string { 22 | return "pytest" 23 | } 24 | 25 | func NewPytest(c RunnerConfig) Pytest { 26 | if !checkPythonPackageInstalled("buildkite_test_collector") { // python import only use underscore 27 | fmt.Fprintln(os.Stderr, "Error: Required Python package 'buildkite-test-collector' is not installed.") 28 | fmt.Fprintln(os.Stderr, "Please install it with: pip install buildkite-test-collector.") 29 | os.Exit(1) 30 | } 31 | 32 | if c.TestCommand == "" { 33 | c.TestCommand = "pytest {{testExamples}} --json={{resultPath}}" 34 | } 35 | 36 | if c.TestFilePattern == "" { 37 | c.TestFilePattern = "**/{*_test,test_*}.py" 38 | } 39 | 40 | if c.RetryTestCommand == "" { 41 | c.RetryTestCommand = c.TestCommand 42 | } 43 | 44 | if c.ResultPath == "" { 45 | c.ResultPath = getRandomTempFilename() 46 | } 47 | 48 | return Pytest{ 49 | RunnerConfig: c, 50 | } 51 | } 52 | 53 | func (p Pytest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error { 54 | testPaths := make([]string, len(testCases)) 55 | for i, tc := range testCases { 56 | testPaths[i] = tc.Path 57 | } 58 | 59 | command := p.TestCommand 60 | 61 | if retry { 62 | command = p.RetryTestCommand 63 | } 64 | 65 | cmdName, cmdArgs, err := p.commandNameAndArgs(command, testPaths) 66 | if err != nil { 67 | return fmt.Errorf("failed to build command: %w", err) 68 | } 69 | 70 | cmd := exec.Command(cmdName, cmdArgs...) 71 | 72 | err = runAndForwardSignal(cmd) 73 | 74 | // Only rescue exit code 1 because it indicates a test failures. 75 | // Ref: https://docs.pytest.org/en/7.1.x/reference/exit-codes.html 76 | if exitError := new(exec.ExitError); errors.As(err, &exitError) && exitError.ExitCode() != 1 { 77 | return err 78 | } 79 | 80 | tests, parseErr := ParsePytestCollectorResult(p.ResultPath) 81 | 82 | if parseErr != nil { 83 | fmt.Println("Buildkite Test Engine Client: Failed to read json output, failed tests will not be retried.") 84 | return err 85 | } 86 | 87 | for _, test := range tests { 88 | 89 | result.RecordTestResult(plan.TestCase{ 90 | Identifier: test.Id, 91 | Format: plan.TestCaseFormatExample, 92 | Scope: test.Scope, 93 | Name: test.Name, 94 | // pytest can execute individual test using node id, which is a filename, classname (if any), and function, separated by `::`. 95 | // Ref: https://docs.pytest.org/en/6.2.x/usage.html#nodeids 96 | Path: fmt.Sprintf("%s::%s", test.Scope, test.Name), 97 | }, test.Result) 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func (p Pytest) GetFiles() ([]string, error) { 104 | debug.Println("Discovering test files with include pattern:", p.TestFilePattern, "exclude pattern:", p.TestFileExcludePattern) 105 | files, err := discoverTestFiles(p.TestFilePattern, p.TestFileExcludePattern) 106 | debug.Println("Discovered", len(files), "files") 107 | 108 | if err != nil { 109 | return nil, err 110 | } 111 | 112 | if len(files) == 0 { 113 | return nil, fmt.Errorf("no files found with pattern %q and exclude pattern %q", p.TestFilePattern, p.TestFileExcludePattern) 114 | } 115 | 116 | return files, nil 117 | } 118 | 119 | func (p Pytest) GetExamples(files []string) ([]plan.TestCase, error) { 120 | return nil, fmt.Errorf("not supported in pytest") 121 | } 122 | 123 | func (p Pytest) commandNameAndArgs(cmd string, testCases []string) (string, []string, error) { 124 | testExamples := strings.Join(testCases, " ") 125 | 126 | if strings.Contains(cmd, "{{testExamples}}") { 127 | cmd = strings.Replace(cmd, "{{testExamples}}", testExamples, 1) 128 | } else { 129 | cmd = cmd + " " + testExamples 130 | } 131 | 132 | cmd = strings.Replace(cmd, "{{resultPath}}", p.ResultPath, 1) 133 | 134 | args, err := shellquote.Split(cmd) 135 | 136 | if err != nil { 137 | return "", []string{}, err 138 | } 139 | 140 | return args[0], args[1:], nil 141 | } 142 | 143 | // TestEngineTest represents a Test Engine test result object. 144 | // Some attributes such as `history` and `failure_reason` are omitted as they are not needed by bktec. 145 | // Ref: https://buildkite.com/docs/test-engine/importing-json#json-test-results-data-reference-test-result-objects 146 | // 147 | // Currently, only pytest uses result from test collector. If we use this somewhere else in the future, we may want to extract this struct. 148 | type TestEngineTest struct { 149 | Id string 150 | Name string 151 | Scope string 152 | Location string 153 | FileName string `json:"file_name,omitempty"` 154 | Result TestStatus 155 | } 156 | 157 | func ParsePytestCollectorResult(path string) ([]TestEngineTest, error) { 158 | var results []TestEngineTest 159 | data, err := os.ReadFile(path) 160 | if err != nil { 161 | return nil, fmt.Errorf("failed to read json: %v", err) 162 | } 163 | 164 | if err := json.Unmarshal(data, &results); err != nil { 165 | return nil, fmt.Errorf("failed to parse json: %v", err) 166 | } 167 | 168 | return results, nil 169 | } 170 | 171 | func getRandomTempFilename() string { 172 | tempDir, err := os.MkdirTemp("", "bktec-pytest-*") 173 | if err != nil { 174 | panic(err) 175 | } 176 | return filepath.Join(tempDir, "pytest-results.json") 177 | } 178 | 179 | func checkPythonPackageInstalled(pkgName string) bool { 180 | // This is the most reliable way I can find. Hopefully it should work regardless of if user uses pip, poetry or uv 181 | pythonCmd := exec.Command("python", "-c", "import importlib.util, sys; print(importlib.util.find_spec(sys.argv[1]) is not None)", pkgName) 182 | output, err := pythonCmd.Output() 183 | 184 | return err == nil && strings.TrimSpace(string(output)) == "True" 185 | } 186 | -------------------------------------------------------------------------------- /internal/runner/pytest_pants.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "strings" 9 | 10 | "github.com/buildkite/test-engine-client/internal/plan" 11 | "github.com/kballard/go-shellquote" 12 | ) 13 | 14 | type PytestPants struct { 15 | RunnerConfig 16 | } 17 | 18 | func (p PytestPants) Name() string { 19 | return "pytest-pants" 20 | } 21 | 22 | func NewPytestPants(c RunnerConfig) PytestPants { 23 | fmt.Fprintln(os.Stderr, "Info: Python package 'buildkite-test-collector' is required and will not be verified by bktec. Please ensure it is added to the pants resolve used by pytest. See https://github.com/buildkite/test-engine-client/blob/main/docs/pytest-pants.md for more information.") 24 | 25 | if c.TestCommand == "" { 26 | fmt.Fprintln(os.Stderr, "Error: The test command must be set via BUILDKITE_TEST_ENGINE_TEST_CMD.") 27 | os.Exit(1) 28 | } 29 | 30 | if c.TestFilePattern != "" || c.TestFileExcludePattern != "" { 31 | fmt.Fprintln(os.Stderr, "Warning: Pants test runner variant does not support discovering test files. Please ensure the test command is set correctly via BUILDKITE_TEST_ENGINE_TEST_CMD and do *not* set either:") 32 | fmt.Fprintf(os.Stderr, " BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=%q\n", c.TestFilePattern) 33 | fmt.Fprintf(os.Stderr, " BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=%q\n", c.TestFileExcludePattern) 34 | } 35 | 36 | if c.TestFilePattern == "" { 37 | c.TestFilePattern = "**/{*_test,test_*}.py" 38 | } 39 | 40 | if c.RetryTestCommand == "" { 41 | c.RetryTestCommand = c.TestCommand 42 | } 43 | 44 | if c.ResultPath == "" { 45 | c.ResultPath = getRandomTempFilename() 46 | } 47 | 48 | return PytestPants{ 49 | RunnerConfig: c, 50 | } 51 | } 52 | 53 | func (p PytestPants) Run(result *RunResult, testCases []plan.TestCase, retry bool) error { 54 | testPaths := make([]string, len(testCases)) 55 | for i, tc := range testCases { 56 | testPaths[i] = tc.Path 57 | } 58 | 59 | command := p.TestCommand 60 | 61 | if retry { 62 | command = p.RetryTestCommand 63 | } 64 | 65 | cmdName, cmdArgs, err := p.commandNameAndArgs(command, testPaths) 66 | if err != nil { 67 | return fmt.Errorf("failed to build command: %w", err) 68 | } 69 | 70 | cmd := exec.Command(cmdName, cmdArgs...) 71 | 72 | err = runAndForwardSignal(cmd) 73 | 74 | // Only rescue exit code 1 because it indicates a test failures. 75 | // Ref: https://docs.pytest.org/en/7.1.x/reference/exit-codes.html 76 | if exitError := new(exec.ExitError); errors.As(err, &exitError) && exitError.ExitCode() != 1 { 77 | return err 78 | } 79 | 80 | tests, parseErr := ParsePytestCollectorResult(p.ResultPath) 81 | 82 | if parseErr != nil { 83 | fmt.Println("Buildkite Test Engine Client: Failed to read json output, failed tests will not be retried.") 84 | return err 85 | } 86 | 87 | for _, test := range tests { 88 | 89 | result.RecordTestResult(plan.TestCase{ 90 | Identifier: test.Id, 91 | Format: plan.TestCaseFormatExample, 92 | Scope: test.Scope, 93 | Name: test.Name, 94 | // pytest can execute individual test using node id, which is a filename, classname (if any), and function, separated by `::`. 95 | // Ref: https://docs.pytest.org/en/6.2.x/usage.html#nodeids 96 | Path: fmt.Sprintf("%s::%s", test.Scope, test.Name), 97 | }, test.Result) 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func (p PytestPants) GetFiles() ([]string, error) { 104 | return []string{}, nil 105 | } 106 | 107 | func (p PytestPants) GetExamples(files []string) ([]plan.TestCase, error) { 108 | return nil, fmt.Errorf("not supported in pytest pants") 109 | } 110 | 111 | func (p PytestPants) commandNameAndArgs(cmd string, testCases []string) (string, []string, error) { 112 | if strings.Contains(cmd, "{{testExamples}}") { 113 | return "", []string{}, fmt.Errorf("currently, bktec does not support dynamically injecting {{testExamples}}. Please ensure the test command in BUILDKITE_TEST_ENGINE_TEST_CMD does *not* include {{testExamples}}") 114 | } 115 | 116 | // Split command into parts before and after the first -- 117 | parts := strings.SplitN(cmd, "--", 2) 118 | if len(parts) != 2 { 119 | return "", []string{}, fmt.Errorf("please ensure the test command in BUILDKITE_TEST_ENGINE_TEST_CMD includes a -- separator") 120 | } 121 | 122 | // Check that both required flags are after the -- 123 | afterDash := parts[1] 124 | if !strings.Contains(afterDash, "--json={{resultPath}}") { 125 | return "", []string{}, fmt.Errorf("please ensure the test command in BUILDKITE_TEST_ENGINE_TEST_CMD includes --json={{resultPath}} after the -- separator") 126 | } 127 | 128 | if !strings.Contains(afterDash, "--merge-json") { 129 | return "", []string{}, fmt.Errorf("please ensure the test command in BUILDKITE_TEST_ENGINE_TEST_CMD includes --merge-json after the -- separator") 130 | } 131 | 132 | cmd = strings.Replace(cmd, "{{resultPath}}", p.ResultPath, 1) 133 | 134 | args, err := shellquote.Split(cmd) 135 | 136 | if err != nil { 137 | return "", []string{}, err 138 | } 139 | 140 | return args[0], args[1:], nil 141 | } 142 | -------------------------------------------------------------------------------- /internal/runner/run_result.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/buildkite/test-engine-client/internal/plan" 5 | ) 6 | 7 | type RunStatus string 8 | 9 | const ( 10 | // RunStatusPassed indicates that the run was completed and all tests passed. 11 | RunStatusPassed RunStatus = "passed" 12 | // RunStatusFailed indicates that the run was completed, but one or more tests failed. 13 | RunStatusFailed RunStatus = "failed" 14 | // RunStatusError indicates that the run was completed, but there was an error outside of the tests. 15 | RunStatusError RunStatus = "error" 16 | // RunStatusUnknown indicates that the run status is unknown. 17 | // It could be that no tests were run, run was interrupted, or the report is not available. 18 | RunStatusUnknown RunStatus = "unknown" 19 | ) 20 | 21 | // RunResult is a struct to keep track the results of a test run. 22 | // It contains the logics to record test results, calculate the status of the run. 23 | type RunResult struct { 24 | // tests is a map containing individual test results. 25 | tests map[string]*TestResult 26 | // mutedTestLookup is a map containing the test identifiers of muted tests. 27 | // This list might contain tests that are not part of the current run (i.e. belong to a different node). 28 | mutedTestLookup map[string]bool 29 | error error 30 | } 31 | 32 | func NewRunResult(mutedTests []plan.TestCase) *RunResult { 33 | r := &RunResult{ 34 | tests: make(map[string]*TestResult), 35 | mutedTestLookup: make(map[string]bool), 36 | } 37 | 38 | for _, testCase := range mutedTests { 39 | identifier := mutedTestIdentifier(testCase) 40 | r.mutedTestLookup[identifier] = true 41 | } 42 | return r 43 | } 44 | 45 | // getTest finds or creates a TestResult struct for a given test case 46 | // in the tests map, and returns a pointer to it. 47 | func (r *RunResult) getTest(testCase plan.TestCase) *TestResult { 48 | if r.tests == nil { 49 | r.tests = make(map[string]*TestResult) 50 | } 51 | 52 | testIdentifier := testIdentifier(testCase) 53 | 54 | test, exists := r.tests[testIdentifier] 55 | if !exists { 56 | test = &TestResult{ 57 | TestCase: testCase, 58 | } 59 | r.tests[testIdentifier] = test 60 | } 61 | return test 62 | } 63 | 64 | // RecordTestResult records the result of a test case. 65 | // If the test case found in the mutedTestLookup, it will be marked as muted. 66 | func (r *RunResult) RecordTestResult(testCase plan.TestCase, status TestStatus) { 67 | test := r.getTest(testCase) 68 | test.Status = status 69 | test.ExecutionCount++ 70 | if r.mutedTestLookup[mutedTestIdentifier(testCase)] { 71 | test.Muted = true 72 | } 73 | } 74 | 75 | // FailedTests returns a list of test cases that failed. 76 | func (r *RunResult) FailedTests() []plan.TestCase { 77 | var failedTests []plan.TestCase 78 | 79 | for _, test := range r.tests { 80 | if test.Status == TestStatusFailed && !test.Muted { 81 | failedTests = append(failedTests, test.TestCase) 82 | } 83 | } 84 | 85 | return failedTests 86 | } 87 | 88 | func (r *RunResult) MutedTests() []TestResult { 89 | var mutedTests []TestResult 90 | for _, test := range r.tests { 91 | if test.Muted { 92 | mutedTests = append(mutedTests, *test) 93 | } 94 | } 95 | 96 | return mutedTests 97 | } 98 | 99 | func (r *RunResult) SkippedTests() []plan.TestCase { 100 | var skippedTests []plan.TestCase 101 | 102 | for _, test := range r.tests { 103 | if test.Status == TestStatusSkipped { 104 | skippedTests = append(skippedTests, test.TestCase) 105 | } 106 | } 107 | return skippedTests 108 | } 109 | 110 | func (r *RunResult) FailedMutedTests() []plan.TestCase { 111 | var failedTests []plan.TestCase 112 | 113 | for _, test := range r.tests { 114 | if test.Status == TestStatusFailed && test.Muted { 115 | failedTests = append(failedTests, test.TestCase) 116 | } 117 | } 118 | return failedTests 119 | } 120 | 121 | // Status returns the overall status of the test run. 122 | // If there is an error, it returns RunStatusError. 123 | // If there are failed tests, it returns RunStatusFailed. 124 | // Otherwise, it returns RunStatusPassed. 125 | func (r *RunResult) Status() RunStatus { 126 | if r.error != nil { 127 | return RunStatusError 128 | } 129 | 130 | if len(r.tests) == 0 { 131 | return RunStatusUnknown 132 | } 133 | 134 | if len(r.FailedTests()) > 0 { 135 | return RunStatusFailed 136 | } 137 | 138 | return RunStatusPassed 139 | } 140 | 141 | func (r *RunResult) Error() error { 142 | return r.error 143 | } 144 | 145 | type RunStatistics struct { 146 | Total int `json:"total"` 147 | PassedOnFirstRun int `json:"passed_on_first_run"` 148 | PassedOnRetry int `json:"passed_on_retry"` 149 | MutedPassed int `json:"muted_passed"` 150 | MutedFailed int `json:"muted_failed"` 151 | Failed int `json:"failed"` 152 | Skipped int `json:"skipped"` 153 | } 154 | 155 | func (r *RunResult) Statistics() RunStatistics { 156 | stat := &RunStatistics{} 157 | 158 | for _, testResult := range r.tests { 159 | switch { 160 | case testResult.Muted: 161 | switch testResult.Status { 162 | case TestStatusPassed: 163 | stat.MutedPassed++ 164 | case TestStatusFailed: 165 | stat.MutedFailed++ 166 | } 167 | 168 | case testResult.Status == TestStatusPassed: 169 | if testResult.ExecutionCount > 1 { 170 | stat.PassedOnRetry++ 171 | } else { 172 | stat.PassedOnFirstRun++ 173 | } 174 | 175 | case testResult.Status == TestStatusFailed: 176 | stat.Failed++ 177 | case testResult.Status == TestStatusSkipped: 178 | stat.Skipped++ 179 | } 180 | } 181 | 182 | stat.Total = len(r.tests) 183 | 184 | return *stat 185 | } 186 | -------------------------------------------------------------------------------- /internal/runner/signal_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package runner 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | ) 9 | 10 | // isIgnoredSignal checks if the signal should be ignored. 11 | // On Unix-like systems, we ignore SIGCHLD, which is sent when a child process terminates. 12 | var isIgnoredSignal = func(sig os.Signal) bool { 13 | return sig == syscall.SIGCHLD 14 | } 15 | -------------------------------------------------------------------------------- /internal/runner/signal_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | 3 | package runner 4 | 5 | import "os" 6 | 7 | // isIgnoredSignal checks if the signal should be ignored. 8 | // On Windows, there isn't a direct equivalent to SIGCHLD that needs ignoring in this context. 9 | var isIgnoredSignal = func(sig os.Signal) bool { 10 | return false 11 | } 12 | -------------------------------------------------------------------------------- /internal/runner/test_result.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import "github.com/buildkite/test-engine-client/internal/plan" 4 | 5 | type TestStatus string 6 | 7 | const ( 8 | TestStatusPassed TestStatus = "passed" 9 | TestStatusFailed TestStatus = "failed" 10 | TestStatusSkipped TestStatus = "skipped" 11 | ) 12 | 13 | // TestResult is a struct to keep track the result of an individual test case. 14 | type TestResult struct { 15 | plan.TestCase 16 | Status TestStatus 17 | ExecutionCount int 18 | Muted bool 19 | } 20 | 21 | // testIdentifier returns a unique identifier for a test case based on its scope, name and path. 22 | // Different tests can have the same name and scope, therefore the path is included in the identifier 23 | // to make it unique. 24 | func testIdentifier(testCase plan.TestCase) string { 25 | return testCase.Scope + "/" + testCase.Name + "/" + testCase.Path 26 | } 27 | 28 | // mutedTestIdentifier returns a unique identifier for a muted test case based on its scope and name. 29 | // Test Engine server identify a unique tests by its scope and name only, therefore we need follow the same logic 30 | // to match a local test with the list of muted tests received from the server. 31 | func mutedTestIdentifier(testCase plan.TestCase) string { 32 | return testCase.Scope + "/" + testCase.Name 33 | } 34 | -------------------------------------------------------------------------------- /internal/runner/testdata/cypress/cypress.config.js: -------------------------------------------------------------------------------- 1 | const { defineConfig } = require('cypress'); 2 | 3 | module.exports = defineConfig({ 4 | e2e: { 5 | supportFile: false, 6 | }, 7 | screenshotOnRunFailure: false, 8 | video: false, 9 | }) 10 | -------------------------------------------------------------------------------- /internal/runner/testdata/cypress/cypress/e2e/failing_spec.cy.js: -------------------------------------------------------------------------------- 1 | describe('Failing spec', () => { 2 | it('fails', () => { 3 | expect(true).to.be.false 4 | }) 5 | }) 6 | -------------------------------------------------------------------------------- /internal/runner/testdata/cypress/cypress/e2e/flaky_spec.cy.js: -------------------------------------------------------------------------------- 1 | describe("Flaky spec", () => { 2 | it("is 50% flaky", () => { 3 | expect(Math.random() > 0.5).to.be.true; 4 | }); 5 | }); 6 | -------------------------------------------------------------------------------- /internal/runner/testdata/cypress/cypress/e2e/passing_spec.cy.js: -------------------------------------------------------------------------------- 1 | describe('Passing spec', () => { 2 | beforeEach(() => { 3 | cy.visit('index.html') 4 | }) 5 | 6 | it('has a title', () => { 7 | cy.title().should('eq', 'Buildkite Test Engine Client - Cypress Example') 8 | }) 9 | 10 | it('says hello', () => { 11 | cy.contains('Hello there!') 12 | }) 13 | }) 14 | -------------------------------------------------------------------------------- /internal/runner/testdata/cypress/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Buildkite Test Engine Client - Cypress Example 6 | 7 | 8 | 9 |
10 |

Buildkite Test Engine Client - Cypress Example

11 |

Hello there!

12 |
13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /internal/runner/testdata/cypress/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cypress-example", 3 | "version": "1.0.0", 4 | "scripts": { 5 | "open": "cypress open", 6 | "test": "cypress run" 7 | }, 8 | "devDependencies": { 9 | "cypress": "^12.2.0" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /internal/runner/testdata/files/animals/ant_test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildkite/test-engine-client/1cac00ffcff636a1120254bad584d99c17ece136/internal/runner/testdata/files/animals/ant_test -------------------------------------------------------------------------------- /internal/runner/testdata/files/animals/bee_test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildkite/test-engine-client/1cac00ffcff636a1120254bad584d99c17ece136/internal/runner/testdata/files/animals/bee_test -------------------------------------------------------------------------------- /internal/runner/testdata/files/fruits/apple_test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildkite/test-engine-client/1cac00ffcff636a1120254bad584d99c17ece136/internal/runner/testdata/files/fruits/apple_test -------------------------------------------------------------------------------- /internal/runner/testdata/files/fruits/banana_test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildkite/test-engine-client/1cac00ffcff636a1120254bad584d99c17ece136/internal/runner/testdata/files/fruits/banana_test -------------------------------------------------------------------------------- /internal/runner/testdata/files/vegetable_test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildkite/test-engine-client/1cac00ffcff636a1120254bad584d99c17ece136/internal/runner/testdata/files/vegetable_test -------------------------------------------------------------------------------- /internal/runner/testdata/go/bad/bad_test.go: -------------------------------------------------------------------------------- 1 | package bad 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestBad(t *testing.T) { 8 | t.Errorf("Test failed: expected %v, got %v", true, false) 9 | } 10 | -------------------------------------------------------------------------------- /internal/runner/testdata/go/go.mod: -------------------------------------------------------------------------------- 1 | module example.com/hello 2 | 3 | go 1.21 4 | -------------------------------------------------------------------------------- /internal/runner/testdata/go/hello_test.go: -------------------------------------------------------------------------------- 1 | package hello 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestHelloWorld(t *testing.T) { 8 | // A simple placeholder test 9 | } 10 | -------------------------------------------------------------------------------- /internal/runner/testdata/jest/failure.spec.js: -------------------------------------------------------------------------------- 1 | describe('this will fail', () => { 2 | it('for sure', () => { 3 | expect(1).toEqual(2) 4 | }) 5 | }) 6 | -------------------------------------------------------------------------------- /internal/runner/testdata/jest/jest.config.js: -------------------------------------------------------------------------------- 1 | // This file is intentionally left blank. 2 | // Jest requires a jest.config.js file to run. It can be empty, so it doesn't 3 | // serve much use other than to satisfy Jest for the Jest tests. 4 | const config = { 5 | "testLocationInResults": true, 6 | }; 7 | 8 | module.exports = config; 9 | -------------------------------------------------------------------------------- /internal/runner/testdata/jest/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jest-example", 3 | "version": "1.0.0", 4 | "scripts": { 5 | "test": "jest" 6 | }, 7 | "devDependencies": { 8 | "jest": "^29.7.0" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /internal/runner/testdata/jest/runtimeError.spec.js: -------------------------------------------------------------------------------- 1 | describe('this will fail', () => { 2 | boom() 3 | }) 4 | -------------------------------------------------------------------------------- /internal/runner/testdata/jest/skipped.spec.js: -------------------------------------------------------------------------------- 1 | describe('this will be skipped', () => { 2 | xit('for sure', () => { 3 | expect(1).toEqual(2) 4 | }) 5 | 6 | it.todo('todo yeah') 7 | }) 8 | -------------------------------------------------------------------------------- /internal/runner/testdata/jest/slow.spec.js: -------------------------------------------------------------------------------- 1 | describe('slow test', () => { 2 | it('wait for 2 seconds', async () => { 3 | await new Promise((resolve) => setTimeout(resolve, 2000)); 4 | }) 5 | }) 6 | -------------------------------------------------------------------------------- /internal/runner/testdata/jest/spells/expelliarmus.spec.js: -------------------------------------------------------------------------------- 1 | describe('expelliarmus', () => { 2 | test('disarms the opponent', () => { 3 | expect(1 + 1).toEqual(2) 4 | }) 5 | }); 6 | -------------------------------------------------------------------------------- /internal/runner/testdata/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "javascript-runners", 3 | "private": true, 4 | "workspaces": [ 5 | "cypress", 6 | "jest", 7 | "playwright" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /internal/runner/testdata/playwright/.gitignore: -------------------------------------------------------------------------------- 1 | /test-results/ 2 | /playwright-report/ 3 | /playwright/.cache/ 4 | -------------------------------------------------------------------------------- /internal/runner/testdata/playwright/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Playwright example 6 | 7 | 8 | 9 |

Hello, World!

10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /internal/runner/testdata/playwright/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "playwright-example", 3 | "version": "1.0.0", 4 | "dependencies": { 5 | "http-server": "^14.1.1" 6 | }, 7 | "devDependencies": { 8 | "@playwright/test": "^1.48.0" 9 | }, 10 | "scripts": { 11 | "start": "yarn run http-server -p 8080", 12 | "test": "yarn run playwright test" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /internal/runner/testdata/playwright/playwright.config.js: -------------------------------------------------------------------------------- 1 | const { defineConfig, devices } = require('@playwright/test'); 2 | 3 | /** 4 | * @see https://playwright.dev/docs/test-configuration 5 | */ 6 | module.exports = defineConfig({ 7 | testDir: './tests', 8 | reporter: [ 9 | ['line'], 10 | ['json', { outputFile: './test-results/results.json' }] 11 | ], 12 | webServer: { 13 | command: 'yarn start', 14 | url: 'http://127.0.0.1:8080', 15 | }, 16 | use: { 17 | baseURL: 'http://localhost:8080/', 18 | }, 19 | projects: [ 20 | { 21 | name: 'chromium', 22 | use: { ...devices['Desktop Chrome'] }, 23 | }, 24 | { 25 | name: 'firefox', 26 | use: { ...devices['Desktop Firefox'] }, 27 | }, 28 | ], 29 | }); 30 | 31 | -------------------------------------------------------------------------------- /internal/runner/testdata/playwright/tests/error.spec.js: -------------------------------------------------------------------------------- 1 | import { test, expect } from '@playwright/test'; 2 | 3 | boom(); 4 | 5 | test.describe('test group', () => { 6 | test('failed', () => { 7 | expect(1).toBe(2); 8 | }) 9 | }); 10 | -------------------------------------------------------------------------------- /internal/runner/testdata/playwright/tests/example.spec.js: -------------------------------------------------------------------------------- 1 | import { test, expect } from '@playwright/test'; 2 | 3 | test.describe('home page', () => { 4 | test('has title', async ({ page }) => { 5 | await page.goto('/'); 6 | 7 | await expect(page).toHaveTitle(/Playwright example/); 8 | }); 9 | 10 | test('says hello', async ({ page }) => { 11 | await page.goto('/'); 12 | 13 | const h1 = await page.locator('h1'); 14 | await expect(h1).toHaveText('Hello, World!'); 15 | }) 16 | }); 17 | -------------------------------------------------------------------------------- /internal/runner/testdata/playwright/tests/failed.spec.js: -------------------------------------------------------------------------------- 1 | import { test, expect } from '@playwright/test'; 2 | 3 | 4 | test.describe('test group', () => { 5 | test('failed', () => { 6 | expect(1).toBe(2); 7 | }) 8 | }); 9 | 10 | test('it passes', () => { 11 | expect(true).toBeTruthy(); 12 | }); 13 | 14 | test('timed out', async () => { 15 | test.setTimeout(100); 16 | await new Promise(resolve => setTimeout(resolve, 10000)); 17 | }) 18 | -------------------------------------------------------------------------------- /internal/runner/testdata/playwright/tests/skipped.spec.js: -------------------------------------------------------------------------------- 1 | import { expect, test } from '@playwright/test'; 2 | 3 | 4 | test.skip('it is skipped', () => { 5 | expect(true).toBeTruthy(); 6 | }); 7 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .pytest_cache/ 3 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest/failed_test.py: -------------------------------------------------------------------------------- 1 | def test_failed(): 2 | assert 3 == 5 3 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest/pytest-collector-result.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "db2c4ffa-efee-4b6a-b293-a4c4168aa353", 4 | "scope": "Analytics::Upload associations", 5 | "name": "passed", 6 | "location": "./spec/models/analytics/upload_spec.rb:12", 7 | "file_name": "./spec/models/analytics/upload_spec.rb", 8 | "result": "passed", 9 | "failure_reason": "", 10 | "failure_expanded": [], 11 | "history": { 12 | "start_at": 347611.724809, 13 | "end_at": 347612.451041, 14 | "duration": 0.726232000044547, 15 | "children": [] 16 | } 17 | }, 18 | { 19 | "id": "95f7e024-9e0a-450f-bc64-9edb62d43fa9", 20 | "scope": "Analytics::Upload associations", 21 | "name": "fails", 22 | "location": "./spec/models/analytics/upload_spec.rb:24", 23 | "file_name": "./spec/models/analytics/upload_spec.rb", 24 | "result": "failed", 25 | "failure_reason": "Failure/Error: expect(true).to eq false", 26 | "failure_expanded": [ 27 | { 28 | "expanded": [ 29 | " expected: false", 30 | " got: true", 31 | "", 32 | " (compared using ==)", 33 | "", 34 | " Diff:", 35 | " @@ -1 +1 @@", 36 | " -false", 37 | " +true" 38 | ], 39 | "backtrace": [ 40 | "./spec/models/analytics/upload_spec.rb:25:in `block (3 levels) in '", 41 | "./spec/support/log.rb:17:in `run'", 42 | "./spec/support/log.rb:66:in `block (2 levels) in '", 43 | "./spec/support/database.rb:19:in `block (2 levels) in '", 44 | "/Users/abc/Documents/rspec-buildkite-analytics/lib/rspec/buildkite/analytics/uploader.rb:153:in `block (2 levels) in configure'", 45 | "-e:1:in `
'" 46 | ] 47 | } 48 | ], 49 | "history": { 50 | "start_at": 347611.724809, 51 | "end_at": 347612.451041, 52 | "duration": 0.726232000044547, 53 | "children": [] 54 | } 55 | } 56 | ] 57 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest/result-failed.json: -------------------------------------------------------------------------------- 1 | [{"id": "a1be7e52-0dba-4018-83ce-a1598ca68807", "scope": "tests/failed_test.py", "name": "test_failed", "location": "tests/failed_test.py:0", "file_name": "tests/failed_test.py", "history": {"section": "top", "children": [], "start_at": 2e-05, "end_at": 0.012478, "duration": 0.012458}, "result": "failed", "failure_reason": "def test_failed():\n> assert 3 == 5\nE assert 3 == 5\n\ntests/failed_test.py:2: AssertionError"}] -------------------------------------------------------------------------------- /internal/runner/testdata/pytest/result-passed.json: -------------------------------------------------------------------------------- 1 | [{"id": "66f35b6d-c1a6-4542-a9ea-1fc1f4137663", "scope": "tests/test_sample.py", "name": "test_happy", "location": "tests/test_sample.py:0", "file_name": "tests/test_sample.py", "history": {"section": "top", "children": [], "start_at": 2.4e-05, "end_at": 0.000274, "duration": 0.00025}, "result": "passed"}] -------------------------------------------------------------------------------- /internal/runner/testdata/pytest/test_sample.py: -------------------------------------------------------------------------------- 1 | def test_happy(): 2 | assert 3 == 3 3 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest_pants/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .pytest_cache/ 3 | .pants.d/ 4 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest_pants/3rdparty/python/BUILD: -------------------------------------------------------------------------------- 1 | python_requirements( 2 | name="pytest", 3 | source="pytest-requirements.txt", 4 | resolve="pytest", 5 | overrides={ 6 | "pytest": {"entry_point": "pytest:console_main"}, 7 | }, 8 | ) 9 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest_pants/3rdparty/python/pytest-requirements.txt: -------------------------------------------------------------------------------- 1 | buildkite-test-collector>=1.0.4,<2.0.0 2 | pytest>=8.0.0,<9.0.0 3 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest_pants/BUILD: -------------------------------------------------------------------------------- 1 | python_tests() 2 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest_pants/README.md: -------------------------------------------------------------------------------- 1 | # pytest_pants 2 | 3 | This directory contains a working example of a 4 | [Pants](https://www.pantsbuild.org/) project using test-engine-client. It 5 | demonstrates how to integrate Pants with 6 | [buildkite-test-collector][bk-test-collector] and test-engine-client. 7 | 8 | ## What is Pants? 9 | 10 | [Pants](https://www.pantsbuild.org/) is a fast, scalable, user-friendly build 11 | system for codebases of all sizes. It's particularly useful for: 12 | 13 | - Managing Python dependencies and virtual environments 14 | - Running tests at scale across large codebases 15 | - Incremental builds and testing (only test what changed) 16 | - Enforcing consistent tooling and standards 17 | 18 | ## Key Configuration Files 19 | 20 | This example shows the essential files needed for Pants + pytest integration: 21 | 22 | - **`pants.toml`** - Main Pants configuration file that defines: 23 | - Python version constraints 24 | - Backend plugins (enables Python support) 25 | - Resolve configuration for dependency management 26 | - **`3rdparty/python/BUILD`** - Defines Python requirements as Pants targets 27 | - **`3rdparty/python/pytest-requirements.txt`** - Standard pip requirements file 28 | - **`3rdparty/python/pytest.lock`** - Generated lockfile ensuring reproducible builds 29 | - **`BUILD`** - Tells Pants about Python tests in this directory 30 | 31 | ## Quick Start 32 | 33 | 1. **Install Pants** (if not already installed): 34 | ```sh 35 | curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash 36 | ``` 37 | 38 | 2. **Generate lockfiles** (after adding new dependencies): 39 | ```sh 40 | pants generate-lockfiles --resolve=pytest 41 | ``` 42 | 43 | 3. **Run tests**: 44 | ```sh 45 | pants test :: # Run all tests 46 | pants test //path/to/specific:test # Run specific test 47 | ``` 48 | 49 | ## Integration with test-engine-client 50 | 51 | When using with buildkite-test-collector and test-engine-client: 52 | 53 | - Set `BUILDKITE_TEST_ENGINE_TEST_RUNNER=pytest-pants` 54 | - The `buildkite-test-collector` package must be included in your pytest resolve 55 | - Use pants-specific test commands that include the required `--json` and `--merge-json` flags 56 | 57 | See the main [pytest-pants documentation](../../../docs/pytest-pants.md) for complete integration details. 58 | 59 | ## Updates to pytest pants resolve lock file 60 | 61 | This lock file is what is used by tests. Updating this is particularly useful if the changes being made require a newer version of [buildkite-test-collector][bk-test-collector]. 62 | 63 | ```sh 64 | pants generate-lockfiles --resolve=pytest 65 | ``` 66 | 67 | [bk-test-collector]: https://pypi.org/project/buildkite-test-collector/ 68 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest_pants/failing_test.py: -------------------------------------------------------------------------------- 1 | def test_failed(): 2 | assert 3 == 5 3 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest_pants/pants.toml: -------------------------------------------------------------------------------- 1 | [GLOBAL] 2 | pants_version = "2.26.0" 3 | 4 | backend_packages = [ 5 | "pants.backend.python", 6 | ] 7 | 8 | [python] 9 | interpreter_constraints = [">=3.10,<3.14"] 10 | resolves_generate_lockfiles = true 11 | enable_resolves = true 12 | default_resolve = "pytest" 13 | 14 | 15 | [pytest] 16 | install_from_resolve = "pytest" 17 | requirements = ["//3rdparty/python:pytest"] 18 | 19 | [python.resolves] 20 | pytest = "3rdparty/python/pytest.lock" 21 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest_pants/passing_test.py: -------------------------------------------------------------------------------- 1 | def test_happy(): 2 | assert 3 == 3 3 | -------------------------------------------------------------------------------- /internal/runner/testdata/pytest_pants/result-failed.json: -------------------------------------------------------------------------------- 1 | [{"id": "a1be7e52-0dba-4018-83ce-a1598ca68807", "scope": "tests/failing_test.py", "name": "test_failed", "location": "tests/failing_test.py:0", "file_name": "tests/failing_test.py", "history": {"section": "top", "children": [], "start_at": 2e-05, "end_at": 0.012478, "duration": 0.012458}, "result": "failed", "failure_reason": "def test_failed():\n> assert 3 == 5\nE assert 3 == 5\n\ntests/failing_test.py:2: AssertionError"}] -------------------------------------------------------------------------------- /internal/runner/testdata/rspec/Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rspec" 6 | -------------------------------------------------------------------------------- /internal/runner/testdata/rspec/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | diff-lcs (1.5.1) 5 | rspec (3.13.0) 6 | rspec-core (~> 3.13.0) 7 | rspec-expectations (~> 3.13.0) 8 | rspec-mocks (~> 3.13.0) 9 | rspec-core (3.13.2) 10 | rspec-support (~> 3.13.0) 11 | rspec-expectations (3.13.3) 12 | diff-lcs (>= 1.2.0, < 2.0) 13 | rspec-support (~> 3.13.0) 14 | rspec-mocks (3.13.2) 15 | diff-lcs (>= 1.2.0, < 2.0) 16 | rspec-support (~> 3.13.0) 17 | rspec-support (3.13.2) 18 | 19 | PLATFORMS 20 | arm64-darwin-23 21 | ruby 22 | 23 | DEPENDENCIES 24 | rspec 25 | 26 | BUNDLED WITH 27 | 2.5.16 28 | -------------------------------------------------------------------------------- /internal/runner/testdata/rspec/spec/bad_syntax_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "bad syntax" do 2 | it "is missing an end" do 3 | if true 4 | expect(true).to be_truthy 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /internal/runner/testdata/rspec/spec/failure_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe("Failure") do 2 | it("fails") do 3 | fail 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /internal/runner/testdata/rspec/spec/shared_examples.rb: -------------------------------------------------------------------------------- 1 | RSpec.shared_examples "shared" do 2 | it "behaves like a shared example" do 3 | true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /internal/runner/testdata/rspec/spec/skipped_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe "skipped" do 4 | xit "is skipped" do 5 | expect(true).to be false 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /internal/runner/testdata/rspec/spec/specs_with_shared_examples_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative "./shared_examples.rb" 2 | 3 | RSpec.describe "Specs with shared examples" do 4 | it_behaves_like "shared" 5 | end 6 | -------------------------------------------------------------------------------- /internal/runner/testdata/rspec/spec/spells/expelliarmus_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe("Expelliarmus") do 2 | it("disarms the opponent") do 3 | true 4 | end 5 | 6 | it("knocks the wand out of the opponents hand") do 7 | true 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /internal/runner/testdata/segv.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This is a program that will kill itself with a segmentation fault. 4 | # It is used to simulate a crash in the test runner (e.g. rspec). 5 | kill -SEGV $$ 6 | -------------------------------------------------------------------------------- /internal/runner/util_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | // changeCwd changes the current working directory to the given path for the duration of the test. 9 | // This is useful for tests that need to run in a specific directory, for example to test the runner. 10 | func changeCwd(t *testing.T, path string) { 11 | t.Helper() 12 | origWD, err := os.Getwd() 13 | 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | 18 | err = os.Chdir(path) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | t.Cleanup(func() { 23 | _ = os.Chdir(origWD) 24 | }) 25 | } 26 | -------------------------------------------------------------------------------- /internal/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | // Injected by .goreleaser.yaml during release build. 4 | var Version = "dev" 5 | -------------------------------------------------------------------------------- /packaging/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | COPY bktec /usr/local/bin/bktec 4 | 5 | ENTRYPOINT ["/usr/local/bin/bktec"] 6 | -------------------------------------------------------------------------------- /testdata/retry.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export RETRY=true 4 | 5 | $@ 6 | -------------------------------------------------------------------------------- /testdata/rspec/spec/bad_syntax_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "bad syntax" do 2 | it "is missing an end" do 3 | if true 4 | expect(true).to be_truthy 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /testdata/rspec/spec/fruits/apple_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "Apple" do 2 | it "is red" do 3 | true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /testdata/rspec/spec/fruits/banana_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "Banana" do 2 | it "is yellow" do 3 | true 4 | end 5 | 6 | context "when not ripe" do 7 | it "is green" do 8 | true 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /testdata/rspec/spec/fruits/fig_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "Fig" do 2 | it "is purple" do 3 | true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /testdata/rspec/spec/fruits/tomato_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe "Tomato" do 2 | it "is red" do 3 | true 4 | end 5 | 6 | it "is vegetable" do 7 | if ENV["RETRY"] == "true" 8 | expect(true).to eq(true) 9 | else 10 | expect(true).to eq(false) 11 | end 12 | end 13 | end 14 | --------------------------------------------------------------------------------