├── .bazelignore ├── .bazelrc ├── .bazelversion ├── .bcr ├── metadata.template.json ├── presubmit.yml └── source.template.json ├── .git-blame-ignore-revs ├── .github └── workflows │ ├── ci.yaml │ ├── integration_external_target.yml │ └── integration_no_impacted_targets.yml ├── .gitignore ├── BUILD ├── LICENSE ├── MODULE.bazel ├── MODULE.bazel.lock ├── Makefile ├── README.md ├── WORKSPACE ├── WORKSPACE.bzlmod ├── artifacts.bzl ├── bazel-diff-example.sh ├── cli ├── BUILD └── src │ ├── main │ └── kotlin │ │ └── com │ │ └── bazel_diff │ │ ├── Main.kt │ │ ├── bazel │ │ ├── BazelClient.kt │ │ ├── BazelQueryService.kt │ │ ├── BazelRule.kt │ │ ├── BazelSourceFileTarget.kt │ │ ├── BazelTarget.kt │ │ └── BazelTargetType.kt │ │ ├── cli │ │ ├── BazelDiff.kt │ │ ├── GenerateHashesCommand.kt │ │ ├── GetImpactedTargetsCommand.kt │ │ ├── VersionProvider.kt │ │ └── converter │ │ │ ├── CommaSeparatedValueConverter.kt │ │ │ ├── NormalisingPathConverter.kt │ │ │ └── OptionsConverter.kt │ │ ├── di │ │ └── Modules.kt │ │ ├── extensions │ │ ├── ByteArray.kt │ │ ├── ByteBuffer.kt │ │ └── HashingExtensions.kt │ │ ├── hash │ │ ├── BuildGraphHasher.kt │ │ ├── ExternalRepoResolver.kt │ │ ├── RuleHasher.kt │ │ ├── SourceFileHasher.kt │ │ ├── TargetDigest.kt │ │ ├── TargetHash.kt │ │ └── TargetHasher.kt │ │ ├── interactor │ │ ├── CalculateImpactedTargetsInteractor.kt │ │ ├── DeserialiseHashesInteractor.kt │ │ ├── GenerateHashesInteractor.kt │ │ └── TargetTypeFilter.kt │ │ ├── io │ │ ├── ByteBufferObjectFactory.kt │ │ ├── ByteBufferPool.kt │ │ └── ContentHashProvider.kt │ │ ├── log │ │ └── StderrLogger.kt │ │ └── process │ │ ├── Process.kt │ │ ├── ProcessResult.kt │ │ └── Redirect.kt │ └── test │ ├── kotlin │ └── com │ │ └── bazel_diff │ │ ├── Modules.kt │ │ ├── bazel │ │ └── BazelRuleTest.kt │ │ ├── cli │ │ └── converter │ │ │ ├── CommaSeparatedValueConverterTest.kt │ │ │ ├── NormalisingPathConverterTest.kt │ │ │ └── OptionsConverterTest.kt │ │ ├── e2e │ │ └── E2ETest.kt │ │ ├── hash │ │ ├── BuildGraphHasherTest.kt │ │ ├── FakeSourceFileHasher.kt │ │ ├── HashDiffer.kt │ │ ├── SourceFileHasherTest.kt │ │ ├── TargetHashTest.kt │ │ └── fixture │ │ │ └── foo.ts │ │ ├── interactor │ │ ├── CalculateImpactedTargetsInteractorTest.kt │ │ └── DeserialiseHashesInteractorTest.kt │ │ └── io │ │ ├── ContentHashProviderTest.kt │ │ └── fixture │ │ ├── correct.json │ │ └── wrong.json │ └── resources │ ├── fixture │ ├── cquery-test-android-code-change-android-impacted-targets.txt │ ├── cquery-test-android-code-change-jre-impacted-targets.txt │ ├── cquery-test-android-code-change.zip │ ├── cquery-test-base.zip │ ├── cquery-test-guava-upgrade-android-impacted-targets.txt │ ├── cquery-test-guava-upgrade-jre-impacted-targets.txt │ ├── cquery-test-guava-upgrade.zip │ ├── fine-grained-hash-bzlmod-cquery-test-impacted-targets.txt │ ├── fine-grained-hash-bzlmod-test-1.zip │ ├── fine-grained-hash-bzlmod-test-2.zip │ ├── fine-grained-hash-bzlmod-test-impacted-targets.txt │ ├── fine-grained-hash-external-repo-test-1.zip │ ├── fine-grained-hash-external-repo-test-2.zip │ ├── fine-grained-hash-external-repo-test-impacted-targets.txt │ ├── impacted_targets-1-2-rule-sourcefile.txt │ ├── impacted_targets-1-2.txt │ ├── integration-test-1.zip │ └── integration-test-2.zip │ ├── mockito-extensions │ └── org.mockito.plugins.MockMaker │ └── workspaces │ └── distance_metrics │ ├── A │ ├── BUILD │ ├── one.sh │ └── three.sh │ ├── BUILD │ ├── WORKSPACE │ └── lib.sh ├── constants.bzl ├── demo.gif ├── extensions.bzl ├── maven_install.json └── repositories.bzl /.bazelignore: -------------------------------------------------------------------------------- 1 | cli/src/test/resources/workspaces 2 | -------------------------------------------------------------------------------- /.bazelrc: -------------------------------------------------------------------------------- 1 | run -c opt --show_loading_progress=false --show_progress=false --ui_event_filters=error 2 | run:verbose -c dbg --show_loading_progress=true --show_progress=true --ui_event_filters=info,error,debug 3 | # https://github.com/mockito/mockito/issues/1879 4 | test --sandbox_tmpfs_path=/tmp 5 | -------------------------------------------------------------------------------- /.bazelversion: -------------------------------------------------------------------------------- 1 | 7.3.2 2 | -------------------------------------------------------------------------------- /.bcr/metadata.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "homepage": "https://github.com/Tinder/bazel-diff", 3 | "maintainers": [ 4 | { 5 | "name": "Maxwell Elliott", 6 | "email": "elliott.432@buckeyemail.osu.edu", 7 | "github": "maxwellE" 8 | } 9 | ], 10 | "repository": [ 11 | "github:Tinder/bazel-diff" 12 | ], 13 | "versions": [], 14 | "yanked_versions": {} 15 | } 16 | -------------------------------------------------------------------------------- /.bcr/presubmit.yml: -------------------------------------------------------------------------------- 1 | matrix: 2 | platform: 3 | - debian10 4 | - ubuntu2004 5 | - macos 6 | - macos_arm64 7 | - windows 8 | bazel: 9 | - 7.x 10 | tasks: 11 | verify_targets: 12 | name: Verify build targets 13 | platform: ${{ platform }} 14 | bazel: ${{ bazel }} 15 | build_targets: 16 | - '@bazel-diff//:bazel-diff' 17 | bcr_test_module: 18 | module_path: "" 19 | matrix: 20 | platform: 21 | - ubuntu2004 22 | - macos 23 | - macos_arm64 24 | bazel: 25 | - 7.x 26 | tasks: 27 | run_tests: 28 | name: Run test 29 | platform: ${{ platform }} 30 | bazel: ${{ bazel }} 31 | test_targets: 32 | - '//cli/...' 33 | -------------------------------------------------------------------------------- /.bcr/source.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "integrity": "", 3 | "strip_prefix": "", 4 | "url": "https://github.com/{OWNER}/{REPO}/releases/download/{TAG}/release.tar.gz" 5 | } 6 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | 71bec2274b2c8f2011934d32d6609fcbede8820f 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | test-jre21-bzlmod: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Setup Java JDK 14 | uses: actions/setup-java@v4 15 | with: 16 | distribution: 'temurin' 17 | java-version: '21' 18 | - name: Setup Go environment 19 | uses: actions/setup-go@v5 20 | with: 21 | go-version: ^1.17 22 | id: go 23 | - name: Setup Bazelisk 24 | run: go install github.com/bazelbuild/bazelisk@latest && export PATH=$PATH:$(go env GOPATH)/bin 25 | - uses: actions/checkout@v4 26 | - name: Run bazel-diff tests 27 | run: ~/go/bin/bazelisk coverage --combined_report=lcov //cli/... --enable_bzlmod=true 28 | test-jre21: 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Setup Java JDK 32 | uses: actions/setup-java@v4 33 | with: 34 | distribution: 'temurin' 35 | java-version: '21' 36 | - name: Setup Go environment 37 | uses: actions/setup-go@v5 38 | with: 39 | go-version: ^1.17 40 | id: go 41 | - name: Setup Bazelisk 42 | run: go install github.com/bazelbuild/bazelisk@latest && export PATH=$PATH:$(go env GOPATH)/bin 43 | - uses: actions/checkout@v4 44 | - name: Run bazel-diff tests 45 | run: ~/go/bin/bazelisk coverage --combined_report=lcov //cli/... --enable_bzlmod=false 46 | test-jre11-run-example: 47 | runs-on: ubuntu-latest 48 | steps: 49 | - name: Setup Java JDK 50 | uses: actions/setup-java@v4 51 | with: 52 | distribution: 'temurin' 53 | java-version: '11' 54 | id: java 55 | - name: Setup Go environment 56 | uses: actions/setup-go@v5 57 | with: 58 | go-version: ^1.17 59 | id: go 60 | - name: Setup Bazelisk 61 | run: go install github.com/bazelbuild/bazelisk@latest && export PATH=$PATH:$(go env GOPATH)/bin 62 | - uses: actions/checkout@v4 63 | with: 64 | fetch-depth: 0 65 | - name: Run bazel-diff example script 66 | run: ./bazel-diff-example.sh "$GITHUB_WORKSPACE" ~/go/bin/bazelisk $(git rev-parse HEAD~1) $(git rev-parse HEAD) 67 | deploy: 68 | needs: [test-jre21] 69 | runs-on: ubuntu-latest 70 | strategy: 71 | matrix: 72 | java: [ '11' ] 73 | steps: 74 | - name: Setup Java JDK 75 | uses: actions/setup-java@v4 76 | with: 77 | distribution: 'temurin' 78 | java-version: ${{ matrix.java }} 79 | id: java 80 | - name: Setup Go environment 81 | uses: actions/setup-go@v5 82 | with: 83 | go-version: ^1.17 84 | id: go 85 | - name: Setup Bazelisk 86 | run: go install github.com/bazelbuild/bazelisk@latest && export PATH=$PATH:$(go env GOPATH)/bin 87 | - uses: actions/checkout@v4 88 | - name: Build deployable JAR 89 | run: ~/go/bin/bazelisk build //cli:bazel-diff_deploy.jar 90 | - uses: actions/upload-artifact@v4 91 | with: 92 | name: bazel-diff_deploy.jar 93 | path: bazel-bin/cli/bazel-diff_deploy.jar 94 | if-no-files-found: error 95 | - name: Build release source archive 96 | run: make release_source_archive 97 | - uses: actions/upload-artifact@v4 98 | with: 99 | name: release.tar.gz 100 | path: archives/release.tar.gz 101 | if-no-files-found: error 102 | -------------------------------------------------------------------------------- /.github/workflows/integration_external_target.yml: -------------------------------------------------------------------------------- 1 | name: Integration External Target 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | schedule: 9 | - cron: "0 */12 * * *" 10 | 11 | jobs: 12 | IntegrationExternalTarget: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | java: [ '8', '11' ] 17 | steps: 18 | - name: Setup Java JDK 19 | uses: actions/setup-java@v3 20 | with: 21 | distribution: 'temurin' 22 | java-version: ${{ matrix.java }} 23 | id: java 24 | - uses: actions/checkout@v3 25 | with: 26 | repository: tinder-maxwellelliott/bazel-diff-repro-1 27 | ref: master 28 | fetch-depth: 0 29 | - name: Set bazel version 30 | run: echo "USE_BAZEL_VERSION=7.3.1" > "$HOME/.bazeliskrc" 31 | - name: Run External Target Impact test 32 | run: ./bazel-diff.sh $(pwd) bazel $(git rev-parse HEAD~1) $(git rev-parse HEAD) 33 | - name: Validate Impacted Targets 34 | run: grep -q "//:yo" /tmp/impacted_targets.txt 35 | -------------------------------------------------------------------------------- /.github/workflows/integration_no_impacted_targets.yml: -------------------------------------------------------------------------------- 1 | name: Integration No Impacted Targets 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | schedule: 9 | - cron: "0 */12 * * *" 10 | 11 | jobs: 12 | IntegrationNoImpactedTargets: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | java: [ '8', '11' ] 17 | steps: 18 | - name: Setup Java JDK 19 | uses: actions/setup-java@v3 20 | with: 21 | distribution: 'temurin' 22 | java-version: ${{ matrix.java }} 23 | id: java 24 | - uses: actions/checkout@v3 25 | with: 26 | repository: maxwellE/examples 27 | ref: add_bazel_diff_impact 28 | fetch-depth: 0 29 | - name: Run No Impacted Targets Test 30 | run: cd java-tutorial && ./bazel-diff-example.sh $(pwd) bazel $(git rev-parse HEAD~1) $(git rev-parse HEAD) 31 | - name: Validate Impacted Targets 32 | run: | 33 | if [ -s /tmp/impacted_targets.txt ] 34 | then 35 | echo "Found impacted targets when expected none!" 36 | cat /tmp/impacted_targets.txt 37 | exit 1 38 | fi 39 | 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore backup files. 2 | *~ 3 | # Ignore Vim swap files. 4 | .*.swp 5 | # Ignore files generated by IDEs. 6 | /.classpath 7 | /.factorypath 8 | /.idea/ 9 | /.ijwb/ 10 | /.project 11 | /.settings 12 | /.vscode/ 13 | /bazel.iml 14 | *.iml 15 | # Ignore all bazel-* symlinks. There is no full list since this can change 16 | # based on the name of the directory bazel is cloned into. 17 | /bazel-* 18 | /out 19 | bazel-* 20 | out 21 | # Ignore outputs generated during Bazel bootstrapping. 22 | /output/ 23 | # Bazelisk version file 24 | .bazelversion 25 | # User-specific .bazelrc 26 | user.bazelrc 27 | !bazel-diff-example.sh 28 | .DS_Store 29 | !.bazelversion 30 | archives/ 31 | -------------------------------------------------------------------------------- /BUILD: -------------------------------------------------------------------------------- 1 | alias( 2 | name = "bazel-diff", 3 | actual = "//cli:bazel-diff", 4 | ) 5 | 6 | alias( 7 | name = "format", 8 | actual = "//cli:format", 9 | ) 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, Match Group, LLC 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of Match Group, LLC nor the names of its contributors 12 | may be used to endorse or promote products derived from this software 13 | without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL MATCH GROUP, LLC BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /MODULE.bazel: -------------------------------------------------------------------------------- 1 | module( 2 | name = "bazel-diff", 3 | version = "9.0.4", 4 | compatibility_level = 0, 5 | ) 6 | 7 | bazel_dep(name = "rules_license", version = "1.0.0") 8 | bazel_dep(name = "aspect_rules_lint", version = "1.0.2") 9 | bazel_dep(name = "bazel_skylib", version = "1.6.1") 10 | bazel_dep(name = "rules_proto", version = "7.1.0") 11 | bazel_dep(name = "rules_java", version = "8.11.0") 12 | bazel_dep(name = "rules_kotlin", version = "2.1.3") 13 | bazel_dep(name = "rules_jvm_external", version = "6.7") 14 | 15 | maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") 16 | maven.install( 17 | name = "bazel_diff_maven", 18 | artifacts = [ 19 | "com.google.code.gson:gson:jar:2.9.0", 20 | "com.google.guava:guava:31.1-jre", 21 | "com.willowtreeapps.assertk:assertk-jvm:0.25", 22 | "info.picocli:picocli:jar:4.6.3", 23 | "io.insert-koin:koin-core-jvm:3.1.6", 24 | "io.insert-koin:koin-test-junit4:4.0.0", 25 | "junit:junit:5.11.2", 26 | "org.apache.commons:commons-pool2:2.11.1", 27 | "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2", 28 | "org.mockito.kotlin:mockito-kotlin:5.4.0", 29 | ], 30 | lock_file = "//:maven_install.json", 31 | ) 32 | use_repo( 33 | maven, 34 | bazel_diff_maven = "bazel_diff_maven", 35 | ) 36 | 37 | non_module_repositories = use_extension("//:extensions.bzl", "non_module_repositories") 38 | use_repo(non_module_repositories, "ktfmt") 39 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: release_source_archive 2 | release_source_archive: 3 | mkdir -p archives 4 | tar --exclude-vcs \ 5 | --exclude=bazel-* \ 6 | --exclude=.github \ 7 | --exclude=archives \ 8 | -zcf "archives/release.tar.gz" . 9 | 10 | .PHONY: release_deploy_jar 11 | release_deploy_jar: 12 | bazelisk \ 13 | build \ 14 | //cli:bazel-diff_deploy.jar \ 15 | -c opt 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bazel-diff 2 | 3 | [![Build status](https://github.com/Tinder/bazel-diff/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/Tinder/bazel-diff/actions/workflows/ci.yaml) 4 | 5 | `bazel-diff` is a command line tool for Bazel projects that allows users to determine the exact affected set of impacted targets between two Git revisions. Using this set, users can test or build the exact modified set of targets. 6 | 7 | `bazel-diff` offers several key advantages over rolling your own target diffing solution 8 | 9 | 1. `bazel-diff` is designed for very large Bazel projects. We use Java Protobuf's `parseDelimitedFrom` method alongside Bazel Query's `streamed_proto` output option. These two together allow you to parse Gigabyte or larger protobuf messages. We have tested it with projects containing tens of thousands of targets. 10 | 2. We avoid usage of large command line query lists when interacting with Bazel, [issue here](https://github.com/bazelbuild/bazel/issues/8609). When you interact with Bazel with thousands of query parameters you can reach an upper maximum limit, seeing this error `bash: /usr/local/bin/bazel: Argument list too long`. `bazel-diff` is smart enough to avoid these errors. 11 | 3. `bazel-diff` has been tested with file renames, deletions, and modifications. Works on `bzl` files, `WORKSPACE` files, `BUILD` files and regular files 12 | 13 | Track the feature request for target diffing in Bazel [here](https://github.com/bazelbuild/bazel/issues/7962) 14 | 15 | This approach was inspired by the [following BazelConf talk](https://www.youtube.com/watch?v=9Dk7mtIm7_A) by Benjamin Peterson. 16 | 17 | > There are simpler and faster ways to approximate the affected set of targets. 18 | > However an incorrect solution can result in a system you can't trust, 19 | > because tests could be broken at a commit where you didn't select to run them. 20 | > Then you can't rely on green-to-red (or red-to-green) transitions and 21 | > lose much of the value from your CI system as breakages can be discovered 22 | > later on unrelated commits. 23 | 24 | ## Prerequisites 25 | 26 | * Git 27 | * Bazel 3.3.0 or higher 28 | * Java 8 JDK or higher (Bazel requires this) 29 | 30 | ## Getting Started 31 | 32 | To start using `bazel-diff` immediately, simply clone down the repo and then run the example shell script: 33 | 34 | ```terminal 35 | git clone https://github.com/Tinder/bazel-diff.git 36 | cd bazel-diff 37 | ./bazel-diff-example.sh WORKSPACE_PATH BAZEL_PATH START_GIT_REVISION END_GIT_REVISION 38 | ``` 39 | 40 | Here is a breakdown of those arguments: 41 | 42 | * `WORKSPACE_PATH`: Path to directory containing your `WORKSPACE` file in your Bazel project. 43 | * `BAZEL_PATH`: Path to your Bazel executable 44 | * `START_GIT_REVISION`: Starting Git Branch or SHA for your desired commit range 45 | * `END_GIT_REVISION`: Final Git Branch or SHA for your desired commit range 46 | 47 | You can see the example shell script in action below: 48 | 49 | ![Demo](demo.gif) 50 | 51 | Open `bazel-diff-example.sh` to see how this is implemented. This is purely an example use-case, but it is a great starting point to using `bazel-diff`. 52 | 53 | ## How it works 54 | 55 | `bazel-diff` works as follows 56 | 57 | * The previous revision is checked out, then we run `generate-hashes`. This gives us the hashmap representation for the entire Bazel graph, then we write this JSON to a file. 58 | 59 | * Next we checkout the initial revision, then we run `generate-hashes` and write that JSON to a file. Now we have our final hashmap representation for the Bazel graph. 60 | 61 | * We run `bazel-diff` on the starting and final JSON hash filepaths to get our impacted set of targets. This impacted set of targets is written to a file. 62 | 63 | ## Build Graph Distance Metrics 64 | 65 | `bazel-diff` can optionally compute build graph distance metrics between two revisions. This is 66 | useful for understanding the impact of a change on the build graph. Directly impacted targets are 67 | targets that have had their rule attributes or source file dependencies changed. Indirectly impacted 68 | targets are that are impacted only due to a change in one of their target dependencies. 69 | 70 | For each target, the following metrics are computed: 71 | 72 | * `target_distance`: The number of dependency hops that it takes to get from an impacted target to a directly impacted target. 73 | * `package_distance`: The number of dependency hops that cross a package boundary to get from an impacted target to a directly impacted target. 74 | 75 | Build graph distance metrics can be used by downstream tools to power features such as: 76 | 77 | * Only running sanitizers on impacted tests that are in the same package as a directly impacted target. 78 | * Only running large-sized tests that are within a few package hops of a directly impacted target. 79 | * Only running computationally expensive jobs when an impacted target is within a certain distance of a directly impacted target. 80 | 81 | To enable this feature, you must generate a dependency mapping on your final revision when computing hashes, then pass it into the `get-impacted-targets` command. 82 | 83 | ```bash 84 | git checkout BASE_REV 85 | bazel-diff generate-hashes [...] 86 | 87 | git checkout FINAL_REV 88 | bazel-diff generate-hashes --depEdgesFile deps.json [...] 89 | 90 | bazel-diff get-impacted-targets --depEdgesFile deps.json [...] 91 | ``` 92 | 93 | This will produce an impacted targets json list with target label, target distance, and package distance: 94 | 95 | ```text 96 | [ 97 | {"label": "//foo:bar", "targetDistance": 0, "packageDistance": 0}, 98 | {"label": "//foo:baz", "targetDistance": 1, "packageDistance": 0}, 99 | {"label": "//bar:qux", "targetDistance": 1, "packageDistance": 1} 100 | ] 101 | ``` 102 | 103 | ## CLI Interface 104 | 105 | `bazel-diff` Command 106 | 107 | ```terminal 108 | Usage: bazel-diff [-hvV] [COMMAND] 109 | Writes to a file the impacted targets between two Bazel graph JSON files 110 | -h, --help Show this help message and exit. 111 | -v, --verbose Display query string, missing files and elapsed time 112 | -V, --version Print version information and exit. 113 | Commands: 114 | generate-hashes Writes to a file the SHA256 hashes for each Bazel 115 | Target in the provided workspace. 116 | get-impacted-targets Command-line utility to analyze the state of the bazel 117 | build graph 118 | ``` 119 | 120 | ### `generate-hashes` command 121 | 122 | ```terminal 123 | Usage: bazel-diff generate-hashes [-hkvV] [--[no-]useCquery] [-b=] 124 | [--[no-]excludeExternalTargets] 125 | [--[no-]includeTargetType] 126 | [--contentHashPath=] 127 | [-s=] -w= 128 | [-co=]... 129 | [--cqueryCommandOptions= 130 | ]... 131 | [--fineGrainedHashExternalRepos=]... 133 | [-so=]... 134 | Writes to a file the SHA256 hashes for each Bazel Target in the provided 135 | workspace. 136 | The filepath to write the resulting JSON to. 137 | If not specified, the JSON will be written to STDOUT. 138 | 139 | By default the JSON schema is a dictionary of target => SHA-256 values. 140 | Example: 141 | { 142 | "//cli:bazel-diff_deploy.jar": "4ae310f8ad2bc728934e3509b6102ca658e828b9cd668f79990e95c6663f9633" 143 | ... 144 | } 145 | 146 | If --includeTargetType is specified, the JSON schema will include the target type (SourceFile/Rule/GeneratedFile) 147 | Example: 148 | { 149 | "//cli:src/test/resources/fixture/integration-test-1.zip": "SourceFile#c259eba8539f4c14e4536c61975457c2990e090067893f4a2981e7bb5f4ef4cf", 150 | "//external:android_gmaven_r8": "Rule#795f583449a40814c05e1cc5d833002afed8d12bce5b835579c7f139c2462d61", 151 | "//cli:bazel-diff_deploy.jar": "GeneratedFile#4ae310f8ad2bc728934e3509b6102ca658e828b9cd668f79990e95c6663f9633", 152 | ... 153 | } 154 | --[no-]excludeExternalTargets 155 | If true, exclude external targets. This must be 156 | switched on when using `--noenable_workspace` Bazel 157 | command line option. Defaults to `false`. 158 | --[no-]includeTargetType 159 | Whether include target type in the generated JSON or not. 160 | If false, the generate JSON schema is: {"": ""} 161 | If true, the generate JSON schema is: {"": "#" 162 | -tt, --targetType= 163 | The type of targets to filter, available options are SourceFile/Rule/GeneratedFile 164 | Only works if the JSON was generated with `--includeTargetType` enabled. 165 | If not specified, all types of impacted targets will be returned. 166 | -b, --bazelPath= 167 | Path to Bazel binary. If not specified, the Bazel 168 | binary available in PATH will be used. 169 | -co, --bazelCommandOptions= 170 | Additional space separated Bazel command options used 171 | when invoking `bazel query` 172 | --contentHashPath= 173 | Path to content hash json file. It's a map which maps 174 | relative file path from workspace path to its 175 | content hash. Files in this map will skip content 176 | hashing and use provided value 177 | --cqueryCommandOptions= 178 | Additional space separated Bazel command options used 179 | when invoking `bazel cquery`. This flag is has no 180 | effect if `--useCquery`is false. 181 | --fineGrainedHashExternalRepos= 182 | Comma separate list of external repos in which 183 | fine-grained hashes are computed for the targets. 184 | By default, external repos are treated as an opaque 185 | blob. If an external repo is specified here, 186 | bazel-diff instead computes the hash for individual 187 | targets. For example, one wants to specify `@maven` 188 | here if they use rules_jvm_external so that 189 | individual third party dependency change won't 190 | invalidate all targets in the mono repo. Note that 191 | if `--useCquery` is true; or `--useCquery` is false but 192 | `--bazelCommandOptions=--consistent_labels` is provided, 193 | the canonical repo name must be provided, 194 | e.g. `@@maven` or `@@rules_jvm_external~~maven~maven` (bzlmod) 195 | instead of apparent name `@maven` 196 | -h, --help Show this help message and exit. 197 | --ignoredRuleHashingAttributes= 198 | Attributes that should be ignored when hashing rule 199 | targets. 200 | -k, --[no-]keep_going This flag controls if `bazel query` will be executed 201 | with the `--keep_going` flag or not. Disabling this 202 | flag allows you to catch configuration issues in 203 | your Bazel graph, but may not work for some Bazel 204 | setups. Defaults to `true` 205 | -s, --seed-filepaths= 206 | A text file containing a newline separated list of 207 | filepaths, each of these filepaths will be read and 208 | used as a seed for all targets. 209 | -so, --bazelStartupOptions= 210 | Additional space separated Bazel client startup 211 | options used when invoking Bazel 212 | --[no-]useCquery If true, use cquery instead of query when generating 213 | dependency graphs. Using cquery would yield more 214 | accurate build graph at the cost of slower query 215 | execution. When this is set, one usually also wants 216 | to set `--cqueryCommandOptions` to specify a 217 | targeting platform. Note that this flag only works 218 | with Bazel 6.2.0 or above because lower versions 219 | does not support `--query_file` flag. 220 | -v, --verbose Display query string, missing files and elapsed time 221 | -V, --version Print version information and exit. 222 | -w, --workspacePath= 223 | Path to Bazel workspace directory. 224 | ``` 225 | 226 | **Note**: `--useCquery` flag may not work with very large repos due to limitation 227 | of Bazel. You may want to fallback to use normal query mode in that case. 228 | See for more details. 229 | 230 | ### What does the SHA256 value of `generate-hashes` represent? 231 | 232 | `generate-hashes` is a canonical SHA256 value representing all attributes and inputs into a target. These inputs 233 | are the summation of the rule implementation hash, the SHA256 value 234 | for every attribute of the rule and then the summation of the SHA256 value for 235 | all `rule_inputs` using the same exact algorithm. For source_file inputs the 236 | content of the file are converted into a SHA256 value. 237 | 238 | ### `get-impacted-targets` command 239 | 240 | ```terminal 241 | Usage: bazel-diff get-impacted-targets [-v] -fh= 242 | -o= 243 | -tt= 244 | -sh= 245 | Command-line utility to analyze the state of the bazel build graph 246 | -fh, --finalHashes= 247 | The path to the JSON file of target hashes for the final 248 | revision. Run 'generate-hashes' to get this value. 249 | -o, --output= 250 | Filepath to write the impacted Bazel targets to, newline 251 | separated 252 | -sh, --startingHashes= 253 | The path to the JSON file of target hashes for the initial 254 | revision. Run 'generate-hashes' to get this value. 255 | -tt, --targetType= 256 | The type of targets to filter, available options are SourceFile/Rule/GeneratedFile 257 | Only works if the JSON was generated with `--includeTargetType` enabled. 258 | If not specified, all types of impacted targets will be returned. 259 | -v, --verbose 260 | Display query string, missing files and elapsed time 261 | ``` 262 | 263 | ## Installing 264 | 265 | ### Integrate into your project (recommended) 266 | 267 | First, add the following snippet to your project: 268 | 269 | #### Bzlmod snippet 270 | 271 | ```bazel 272 | bazel_dep(name = "bazel-diff", version = "9.0.3") 273 | ``` 274 | 275 | You can now run the tool with: 276 | 277 | ```terminal 278 | bazel run @bazel-diff//cli:bazel-diff 279 | ``` 280 | 281 | #### WORKSPACE snippet 282 | 283 | ```bazel 284 | http_jar = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") 285 | http_jar( 286 | name = "bazel-diff", 287 | urls = [ 288 | "https://github.com/Tinder/bazel-diff/releases/download/7.0.0/bazel-diff_deploy.jar" 289 | ], 290 | sha256 = "0b9e32f9c20e570846b083743fe967ae54d13e2a1f7364983e0a7792979442be", 291 | ) 292 | ``` 293 | 294 | Second, add in your root `BUILD.bazel` file: 295 | 296 | ```bazel 297 | load("@rules_java//java:defs.bzl", "java_binary") 298 | 299 | java_binary( 300 | name = "bazel-diff", 301 | main_class = "com.bazel_diff.Main", 302 | runtime_deps = ["@bazel-diff//cli:bazel-diff"], 303 | ) 304 | ``` 305 | 306 | That's it! You can now run the tool with: 307 | 308 | ```terminal 309 | bazel run //:bazel-diff 310 | ``` 311 | 312 | > Note, in releases prior to 2.0.0 the value for the `main_class` attribute is just `BazelDiff` 313 | 314 | ### Run Via JAR Release 315 | 316 | ```terminal 317 | curl -Lo bazel-diff.jar https://github.com/Tinder/bazel-diff/releases/latest/download/bazel-diff_deploy.jar 318 | java -jar bazel-diff.jar -h 319 | ``` 320 | 321 | ### Build from Source 322 | 323 | After cloning down the repo, you are good to go, Bazel will handle the rest 324 | 325 | To run the project 326 | 327 | ```terminal 328 | bazel run :bazel-diff -- bazel-diff -h 329 | ``` 330 | 331 | #### Debugging (when running from source) 332 | 333 | To run `bazel-diff` with debug logging, run your commands with the `verbose` config like so: 334 | 335 | ```terminal 336 | bazel run :bazel-diff --config=verbose -- bazel-diff -h 337 | ``` 338 | 339 | ### Build your own deployable JAR 340 | 341 | ```terminal 342 | bazel build //cli:bazel-diff_deploy.jar 343 | java -jar bazel-bin/cli/bazel-diff_deploy.jar # This JAR can be run anywhere 344 | ``` 345 | 346 | ### Build from source in your Bazel Project 347 | 348 | Add the following to your `WORKSPACE` file to add the external repositories, replacing the `RELEASE_ARCHIVE_URL` with the archive url of the bazel-diff release you wish to depend on: 349 | 350 | ```bazel 351 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 352 | 353 | http_archive( 354 | name = "bazel-diff", 355 | urls = [ 356 | "RELEASE_ARCHIVE_URL", 357 | ], 358 | sha256 = "UPDATE_ME", 359 | strip_prefix = "UPDATE_ME" 360 | ) 361 | 362 | load("@bazel-diff//:repositories.bzl", "bazel_diff_dependencies") 363 | 364 | bazel_diff_dependencies() 365 | 366 | load("@rules_jvm_external//:defs.bzl", "maven_install") 367 | load("@bazel-diff//:artifacts.bzl", "BAZEL_DIFF_MAVEN_ARTIFACTS") 368 | 369 | maven_install( 370 | name = "bazel_diff_maven", 371 | artifacts = BAZEL_DIFF_MAVEN_ARTIFACTS, 372 | repositories = [ 373 | "http://uk.maven.org/maven2", 374 | "https://jcenter.bintray.com/", 375 | ], 376 | ) 377 | ``` 378 | 379 | Now you can simply run `bazel-diff` from your project: 380 | 381 | ```terminal 382 | bazel run @bazel-diff//cli:bazel-diff -- bazel-diff -h 383 | ``` 384 | 385 | ## Learn More 386 | 387 | Take a look at the following bazelcon talks to learn more about `bazel-diff`: 388 | 389 | * [BazelCon 2023: Improving CI efficiency with Bazel querying and bazel-diff](https://www.youtube.com/watch?v=QYAbmE_1fSo) 390 | * [BazelCon 2024: Not Going the Distance: Filtering Tests by Build Graph Distance](https://youtu.be/Or0o0Q7Zc1w?si=nIIkTH6TP-pcPoRx) 391 | 392 | ## Running the tests 393 | 394 | To run the tests simply run 395 | 396 | ```terminal 397 | bazel test //... 398 | ``` 399 | 400 | ## Versioning 401 | 402 | We use [SemVer](http://semver.org/) for versioning. For the versions available, 403 | see the [tags on this repository](https://github.com/Tinder/bazel-diff/tags). 404 | 405 | ## License 406 | 407 | --- 408 | 409 | ```text 410 | Copyright (c) 2020, Match Group, LLC 411 | All rights reserved. 412 | 413 | Redistribution and use in source and binary forms, with or without 414 | modification, are permitted provided that the following conditions are met: 415 | * Redistributions of source code must retain the above copyright 416 | notice, this list of conditions and the following disclaimer. 417 | * Redistributions in binary form must reproduce the above copyright 418 | notice, this list of conditions and the following disclaimer in the 419 | documentation and/or other materials provided with the distribution. 420 | * Neither the name of Match Group, LLC nor the names of its contributors 421 | may be used to endorse or promote products derived from this software 422 | without specific prior written permission. 423 | 424 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 425 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 426 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 427 | DISCLAIMED. IN NO EVENT SHALL MATCH GROUP, LLC BE LIABLE FOR ANY 428 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 429 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 430 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 431 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 432 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 433 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 434 | ``` 435 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "bazel_diff") 2 | 3 | load("//:repositories.bzl", "bazel_diff_dependencies") 4 | 5 | bazel_diff_dependencies() 6 | 7 | load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") 8 | 9 | bazel_skylib_workspace() 10 | 11 | load("@bazel_features//:deps.bzl", "bazel_features_deps") 12 | 13 | bazel_features_deps() 14 | 15 | load("@rules_python//python:repositories.bzl", "py_repositories") 16 | 17 | py_repositories() 18 | 19 | load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") 20 | 21 | protobuf_deps() 22 | 23 | load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies") 24 | 25 | rules_proto_dependencies() 26 | 27 | load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains") 28 | 29 | rules_proto_toolchains() 30 | 31 | load("@rules_java//java:repositories.bzl", "rules_java_dependencies", "rules_java_toolchains") 32 | 33 | rules_java_dependencies() 34 | 35 | rules_java_toolchains() 36 | 37 | load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") 38 | 39 | rules_jvm_external_deps() 40 | 41 | load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") 42 | 43 | rules_jvm_external_setup() 44 | 45 | load("@rules_jvm_external//:defs.bzl", "maven_install") 46 | load("@rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories") 47 | load("//:artifacts.bzl", "BAZEL_DIFF_MAVEN_ARTIFACTS") 48 | 49 | kotlin_repositories() 50 | 51 | load("@rules_kotlin//kotlin:core.bzl", "kt_register_toolchains") 52 | 53 | kt_register_toolchains() 54 | 55 | maven_install( 56 | name = "bazel_diff_maven", 57 | artifacts = BAZEL_DIFF_MAVEN_ARTIFACTS, 58 | maven_install_json = "//:maven_install.json", 59 | repositories = [ 60 | "https://repo1.maven.org/maven2/", 61 | ], 62 | ) 63 | 64 | load("@bazel_diff_maven//:defs.bzl", "pinned_maven_install") 65 | 66 | pinned_maven_install() 67 | 68 | load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies") 69 | 70 | aspect_bazel_lib_dependencies() 71 | 72 | load( 73 | "@aspect_rules_lint//format:repositories.bzl", 74 | "fetch_ktfmt", 75 | "rules_lint_dependencies", 76 | ) 77 | 78 | rules_lint_dependencies() 79 | 80 | fetch_ktfmt() 81 | -------------------------------------------------------------------------------- /WORKSPACE.bzlmod: -------------------------------------------------------------------------------- 1 | workspace(name = "bazel_diff") 2 | -------------------------------------------------------------------------------- /artifacts.bzl: -------------------------------------------------------------------------------- 1 | """ Artifacts used to build bazel-diff """ 2 | 3 | BAZEL_DIFF_MAVEN_ARTIFACTS = [ 4 | "com.google.code.gson:gson:jar:2.9.0", 5 | "com.google.guava:guava:31.1-jre", 6 | "com.willowtreeapps.assertk:assertk-jvm:0.25", 7 | "info.picocli:picocli:jar:4.6.3", 8 | "io.insert-koin:koin-core-jvm:3.1.6", 9 | "io.insert-koin:koin-test-junit4:3.1.6", 10 | "junit:junit:4.13.2", 11 | "org.apache.commons:commons-pool2:2.11.1", 12 | "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2", 13 | "org.mockito.kotlin:mockito-kotlin:4.0.0", 14 | ] 15 | -------------------------------------------------------------------------------- /bazel-diff-example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # Path to your Bazel WORKSPACE directory 6 | workspace_path=$1 7 | # Path to your Bazel executable 8 | bazel_path=$2 9 | # Starting Revision SHA 10 | previous_revision=$3 11 | # Final Revision SHA 12 | final_revision=$4 13 | 14 | starting_hashes_json="/tmp/starting_hashes.json" 15 | final_hashes_json="/tmp/final_hashes.json" 16 | impacted_targets_path="/tmp/impacted_targets.txt" 17 | bazel_diff="/tmp/bazel_diff" 18 | 19 | "$bazel_path" run :bazel-diff --script_path="$bazel_diff" 20 | 21 | git -C "$workspace_path" checkout "$previous_revision" --quiet 22 | 23 | echo "Generating Hashes for Revision '$previous_revision'" 24 | $bazel_diff generate-hashes -w "$workspace_path" -b "$bazel_path" $starting_hashes_json 25 | 26 | git -C "$workspace_path" checkout "$final_revision" --quiet 27 | 28 | echo "Generating Hashes for Revision '$final_revision'" 29 | $bazel_diff generate-hashes -w "$workspace_path" -b "$bazel_path" $final_hashes_json 30 | 31 | echo "Determining Impacted Targets" 32 | $bazel_diff get-impacted-targets -sh $starting_hashes_json -fh $final_hashes_json -o $impacted_targets_path 33 | 34 | impacted_targets=() 35 | IFS=$'\n' read -d '' -r -a impacted_targets < $impacted_targets_path || true 36 | formatted_impacted_targets=$(IFS=$'\n'; echo "${impacted_targets[*]}") 37 | echo "Impacted Targets between $previous_revision and $final_revision:" 38 | echo "$formatted_impacted_targets" 39 | echo "" 40 | -------------------------------------------------------------------------------- /cli/BUILD: -------------------------------------------------------------------------------- 1 | load("@aspect_rules_lint//format:defs.bzl", "format_multirun") 2 | load("@rules_java//java:defs.bzl", "java_binary") 3 | load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library", "kt_jvm_test") 4 | 5 | config_setting( 6 | name = "enable_debug", 7 | values = { 8 | "compilation_mode": "dbg", 9 | }, 10 | ) 11 | 12 | genrule( 13 | name = "version_file", 14 | outs = ["version"], 15 | cmd = "echo '{version}' > $@".format(version = module_version() if module_version() else "unknown"), 16 | stamp = 1, 17 | ) 18 | 19 | java_binary( 20 | name = "bazel-diff", 21 | jvm_flags = select({ 22 | ":enable_debug": ["-DDEBUG=true"], 23 | "//conditions:default": [], 24 | }), 25 | main_class = "com.bazel_diff.Main", 26 | visibility = ["//visibility:public"], 27 | runtime_deps = [":cli-lib"], 28 | ) 29 | 30 | kt_jvm_library( 31 | name = "cli-lib", 32 | srcs = glob(["src/main/kotlin/**/*.kt"]), 33 | resources = [":version_file"], 34 | deps = [ 35 | "@bazel_diff_maven//:com_google_code_gson_gson", 36 | "@bazel_diff_maven//:com_google_guava_guava", 37 | "@bazel_diff_maven//:info_picocli_picocli", 38 | "@bazel_diff_maven//:io_insert_koin_koin_core_jvm", 39 | "@bazel_diff_maven//:org_apache_commons_commons_pool2", 40 | "@bazel_diff_maven//:org_jetbrains_kotlinx_kotlinx_coroutines_core_jvm", 41 | "@bazel_tools//src/main/protobuf:analysis_v2_java_proto", 42 | "@bazel_tools//src/main/protobuf:build_java_proto", 43 | ], 44 | ) 45 | 46 | kt_jvm_test( 47 | name = "BuildGraphHasherTest", 48 | jvm_flags = [ 49 | "-Dnet.bytebuddy.experimental=true", 50 | ], 51 | test_class = "com.bazel_diff.hash.BuildGraphHasherTest", 52 | runtime_deps = [":cli-test-lib"], 53 | ) 54 | 55 | kt_jvm_test( 56 | name = "TargetHashTest", 57 | test_class = "com.bazel_diff.hash.TargetHashTest", 58 | runtime_deps = [":cli-test-lib"], 59 | ) 60 | 61 | kt_jvm_test( 62 | name = "SourceFileHasherTest", 63 | data = [ 64 | ":src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts", 65 | ], 66 | test_class = "com.bazel_diff.hash.SourceFileHasherTest", 67 | runtime_deps = [":cli-test-lib"], 68 | ) 69 | 70 | kt_jvm_test( 71 | name = "CalculateImpactedTargetsInteractorTest", 72 | test_class = "com.bazel_diff.interactor.CalculateImpactedTargetsInteractorTest", 73 | runtime_deps = [":cli-test-lib"], 74 | ) 75 | 76 | kt_jvm_test( 77 | name = "NormalisingPathConverterTest", 78 | test_class = "com.bazel_diff.cli.converter.NormalisingPathConverterTest", 79 | runtime_deps = [":cli-test-lib"], 80 | ) 81 | 82 | kt_jvm_test( 83 | name = "OptionsConverterTest", 84 | test_class = "com.bazel_diff.cli.converter.OptionsConverterTest", 85 | runtime_deps = [":cli-test-lib"], 86 | ) 87 | 88 | kt_jvm_test( 89 | name = "DeserialiseHashesInteractorTest", 90 | test_class = "com.bazel_diff.interactor.DeserialiseHashesInteractorTest", 91 | runtime_deps = [":cli-test-lib"], 92 | ) 93 | 94 | kt_jvm_test( 95 | name = "BazelRuleTest", 96 | test_class = "com.bazel_diff.bazel.BazelRuleTest", 97 | runtime_deps = [":cli-test-lib"], 98 | ) 99 | 100 | kt_jvm_test( 101 | name = "E2ETest", 102 | timeout = "long", 103 | data = [":workspaces"], 104 | test_class = "com.bazel_diff.e2e.E2ETest", 105 | runtime_deps = [":cli-test-lib"], 106 | ) 107 | 108 | kt_jvm_test( 109 | name = "ContentHashProviderTest", 110 | data = [ 111 | ":src/test/kotlin/com/bazel_diff/io/fixture/correct.json", 112 | ":src/test/kotlin/com/bazel_diff/io/fixture/wrong.json", 113 | ], 114 | test_class = "com.bazel_diff.io.ContentHashProviderTest", 115 | runtime_deps = [ 116 | ":cli-test-lib", 117 | ], 118 | ) 119 | 120 | kt_jvm_library( 121 | name = "cli-test-lib", 122 | testonly = True, 123 | srcs = glob(["src/test/kotlin/**/*.kt"]), 124 | resources = glob(["src/test/resources/**/*"]), 125 | deps = [ 126 | ":cli-lib", 127 | "@bazel_diff_maven//:com_willowtreeapps_assertk_assertk_jvm", 128 | "@bazel_diff_maven//:io_insert_koin_koin_test_junit4", 129 | "@bazel_diff_maven//:io_insert_koin_koin_test_jvm", 130 | "@bazel_diff_maven//:junit_junit", 131 | "@bazel_diff_maven//:org_mockito_kotlin_mockito_kotlin", 132 | ], 133 | ) 134 | 135 | filegroup( 136 | name = "workspaces", 137 | srcs = [ 138 | "src/test/resources/workspaces", 139 | ], 140 | ) 141 | 142 | java_binary( 143 | name = "ktfmt", 144 | main_class = "com.facebook.ktfmt.cli.Main", 145 | runtime_deps = ["@ktfmt//jar"], 146 | ) 147 | 148 | format_multirun( 149 | name = "format", 150 | kotlin = ":ktfmt", 151 | visibility = ["//visibility:public"], 152 | ) 153 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/Main.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff 2 | 3 | import com.bazel_diff.cli.BazelDiff 4 | import kotlin.system.exitProcess 5 | import picocli.CommandLine 6 | 7 | class Main { 8 | companion object { 9 | @JvmStatic 10 | fun main(args: Array) { 11 | val exitCode = CommandLine(BazelDiff()).execute(*args) 12 | exitProcess(exitCode) 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/bazel/BazelClient.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.bazel 2 | 3 | import com.bazel_diff.log.Logger 4 | import java.util.Calendar 5 | import org.koin.core.component.KoinComponent 6 | import org.koin.core.component.inject 7 | 8 | class BazelClient( 9 | private val useCquery: Boolean, 10 | private val fineGrainedHashExternalRepos: Set, 11 | private val excludeExternalTargets: Boolean 12 | ) : KoinComponent { 13 | private val logger: Logger by inject() 14 | private val queryService: BazelQueryService by inject() 15 | 16 | suspend fun queryAllTargets(): List { 17 | val queryEpoch = Calendar.getInstance().getTimeInMillis() 18 | 19 | val repoTargetsQuery = 20 | if (excludeExternalTargets) emptyList() else listOf("//external:all-targets") 21 | val targets = 22 | if (useCquery) { 23 | // Explicitly listing external repos here sometimes causes issues mentioned at 24 | // https://bazel.build/query/cquery#recursive-target-patterns. Hence, we query all 25 | // dependencies with `deps` 26 | // instead. However, we still need to append all "//external:*" targets because 27 | // fine-grained hash 28 | // computation depends on hashing of source files in external repos as well, which is 29 | // limited to repos 30 | // explicitly mentioned in `fineGrainedHashExternalRepos` flag. Therefore, for any repos 31 | // not mentioned there 32 | // we are still relying on the repo-generation target under `//external` to compute the 33 | // hash. 34 | // 35 | // In addition, we must include all source dependencies in this query in order for them to 36 | // show up in 37 | // `configuredRuleInput`. Hence, one must not filter them out with `kind(rule, deps(..))`. 38 | (queryService.query("deps(//...:all-targets)", useCquery = true) + 39 | queryService.query(repoTargetsQuery.joinToString(" + ") { "'$it'" })) 40 | .distinctBy { it.name } 41 | } else { 42 | val buildTargetsQuery = 43 | listOf("//...:all-targets") + 44 | fineGrainedHashExternalRepos.map { "$it//...:all-targets" } 45 | queryService.query((repoTargetsQuery + buildTargetsQuery).joinToString(" + ") { "'$it'" }) 46 | } 47 | val queryDuration = Calendar.getInstance().getTimeInMillis() - queryEpoch 48 | logger.i { "All targets queried in $queryDuration" } 49 | return targets 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/bazel/BazelQueryService.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.bazel 2 | 3 | import com.bazel_diff.log.Logger 4 | import com.bazel_diff.process.Redirect 5 | import com.bazel_diff.process.process 6 | import com.google.devtools.build.lib.analysis.AnalysisProtosV2 7 | import com.google.devtools.build.lib.query2.proto.proto2api.Build 8 | import java.io.File 9 | import java.nio.file.Files 10 | import java.nio.file.Path 11 | import kotlinx.coroutines.ExperimentalCoroutinesApi 12 | import kotlinx.coroutines.runBlocking 13 | import org.koin.core.component.KoinComponent 14 | import org.koin.core.component.inject 15 | 16 | class BazelQueryService( 17 | private val workingDirectory: Path, 18 | private val bazelPath: Path, 19 | private val startupOptions: List, 20 | private val commandOptions: List, 21 | private val cqueryOptions: List, 22 | private val keepGoing: Boolean, 23 | private val noBazelrc: Boolean, 24 | ) : KoinComponent { 25 | private val logger: Logger by inject() 26 | 27 | suspend fun query(query: String, useCquery: Boolean = false): List { 28 | // Unfortunately, there is still no direct way to tell if a target is compatible or not with the 29 | // proto output 30 | // by itself. So we do an extra cquery with the trick at 31 | // https://bazel.build/extending/platforms#cquery-incompatible-target-detection to first find 32 | // all compatible 33 | // targets. 34 | val compatibleTargetSet = 35 | if (useCquery) { 36 | runQuery(query, useCquery = true, outputCompatibleTargets = true).useLines { 37 | it.filter { it.isNotBlank() }.toSet() 38 | } 39 | } else { 40 | emptySet() 41 | } 42 | val outputFile = runQuery(query, useCquery) 43 | 44 | val targets = 45 | outputFile.inputStream().buffered().use { proto -> 46 | if (useCquery) { 47 | val cqueryResult = AnalysisProtosV2.CqueryResult.parseFrom(proto) 48 | cqueryResult.resultsList 49 | .mapNotNull { toBazelTarget(it.target) } 50 | .filter { it.name in compatibleTargetSet } 51 | } else { 52 | mutableListOf() 53 | .apply { 54 | while (true) { 55 | val target = Build.Target.parseDelimitedFrom(proto) ?: break 56 | // EOF 57 | add(target) 58 | } 59 | } 60 | .mapNotNull { toBazelTarget(it) } 61 | } 62 | } 63 | 64 | return targets 65 | } 66 | 67 | @OptIn(ExperimentalCoroutinesApi::class) 68 | private suspend fun runQuery( 69 | query: String, 70 | useCquery: Boolean, 71 | outputCompatibleTargets: Boolean = false 72 | ): File { 73 | val queryFile = Files.createTempFile(null, ".txt").toFile() 74 | queryFile.deleteOnExit() 75 | val outputFile = Files.createTempFile(null, ".bin").toFile() 76 | outputFile.deleteOnExit() 77 | 78 | queryFile.writeText(query) 79 | 80 | val cmd: MutableList = 81 | ArrayList().apply { 82 | add(bazelPath.toString()) 83 | if (noBazelrc) { 84 | add("--bazelrc=/dev/null") 85 | } 86 | addAll(startupOptions) 87 | if (useCquery) { 88 | add("cquery") 89 | if (!outputCompatibleTargets) { 90 | // There is no need to query the transitions when querying for compatible targets. 91 | add("--transitions=lite") 92 | } 93 | } else { 94 | add("query") 95 | } 96 | add("--output") 97 | if (useCquery) { 98 | if (outputCompatibleTargets) { 99 | add("starlark") 100 | add("--starlark:file") 101 | val cqueryOutputFile = Files.createTempFile(null, ".cquery").toFile() 102 | cqueryOutputFile.deleteOnExit() 103 | cqueryOutputFile.writeText( 104 | """ 105 | def format(target): 106 | if providers(target) == None: 107 | return "" 108 | if "IncompatiblePlatformProvider" not in providers(target): 109 | target_repr = repr(target) 110 | if " BazelTarget.Rule(target) 162 | Build.Target.Discriminator.SOURCE_FILE -> BazelTarget.SourceFile(target) 163 | Build.Target.Discriminator.GENERATED_FILE -> BazelTarget.GeneratedFile(target) 164 | else -> { 165 | logger.w { "Unsupported target type in the build graph: ${target.type.name}" } 166 | null 167 | } 168 | } 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/bazel/BazelRule.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.bazel 2 | 3 | import com.bazel_diff.hash.safePutBytes 4 | import com.bazel_diff.hash.sha256 5 | import com.google.devtools.build.lib.query2.proto.proto2api.Build 6 | 7 | // Ignore generator_location when computing a target's hash since it is likely to change and does 8 | // not 9 | // affect a target's generated actions. Internally, Bazel also does this when computing a target's 10 | // hash: 11 | // https://github.com/bazelbuild/bazel/blob/6971b016f1e258e3bb567a0f9fe7a88ad565d8f2/src/main/java/com/google/devtools/build/lib/query2/query/output/SyntheticAttributeHashCalculator.java#L78-L81 12 | private val DEFAULT_IGNORED_ATTRS = arrayOf("generator_location") 13 | 14 | class BazelRule(private val rule: Build.Rule) { 15 | fun digest(ignoredAttrs: Set): ByteArray { 16 | return sha256 { 17 | safePutBytes(rule.ruleClassBytes.toByteArray()) 18 | safePutBytes(rule.nameBytes.toByteArray()) 19 | safePutBytes(rule.skylarkEnvironmentHashCodeBytes.toByteArray()) 20 | for (attribute in rule.attributeList) { 21 | if (!DEFAULT_IGNORED_ATTRS.contains(attribute.name) && 22 | !ignoredAttrs.contains(attribute.name)) 23 | safePutBytes(attribute.toByteArray()) 24 | } 25 | } 26 | } 27 | 28 | fun ruleInputList(useCquery: Boolean, fineGrainedHashExternalRepos: Set): List { 29 | return if (useCquery) { 30 | rule.configuredRuleInputList.map { it.label } + 31 | rule.ruleInputList 32 | .map { ruleInput: String -> 33 | transformRuleInput(fineGrainedHashExternalRepos, ruleInput) 34 | } 35 | // Only keep the non-fine-grained ones because the others are already covered by 36 | // configuredRuleInputList 37 | .filter { it.startsWith("//external:") } 38 | .distinct() 39 | } else { 40 | rule.ruleInputList 41 | } 42 | } 43 | 44 | val name: String = rule.name 45 | 46 | private fun transformRuleInput( 47 | fineGrainedHashExternalRepos: Set, 48 | ruleInput: String 49 | ): String { 50 | if (isNotMainRepo(ruleInput) && 51 | ruleInput.startsWith("@") && 52 | fineGrainedHashExternalRepos.none { ruleInput.startsWith(it) }) { 53 | val splitRule = ruleInput.split("//".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() 54 | if (splitRule.size == 2) { 55 | var externalRule = splitRule[0] 56 | externalRule = externalRule.replaceFirst("@+".toRegex(), "") 57 | return String.format("//external:%s", externalRule) 58 | } 59 | } 60 | return ruleInput 61 | } 62 | 63 | private fun isNotMainRepo(ruleInput: String): Boolean { 64 | return !ruleInput.startsWith("//") && 65 | !ruleInput.startsWith("@//") && 66 | !ruleInput.startsWith("@@//") 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/bazel/BazelSourceFileTarget.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.bazel 2 | 3 | class BazelSourceFileTarget( 4 | val name: String, 5 | val seed: ByteArray, 6 | ) 7 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/bazel/BazelTarget.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.bazel 2 | 3 | import com.google.devtools.build.lib.query2.proto.proto2api.Build 4 | 5 | sealed class BazelTarget(private val target: Build.Target) { 6 | class SourceFile(target: Build.Target) : BazelTarget(target) { 7 | init { 8 | assert(target.hasSourceFile()) 9 | } 10 | 11 | val sourceFileName: String = target.sourceFile.name 12 | val subincludeList: List = target.sourceFile.subincludeList 13 | override val name: String 14 | get() = sourceFileName 15 | } 16 | 17 | class Rule(target: Build.Target) : BazelTarget(target) { 18 | init { 19 | assert(target.hasRule()) 20 | } 21 | 22 | val rule: BazelRule = BazelRule(target.rule) 23 | override val name: String 24 | get() = rule.name 25 | } 26 | 27 | class GeneratedFile(target: Build.Target) : BazelTarget(target) { 28 | init { 29 | assert(target.hasGeneratedFile()) 30 | } 31 | 32 | val generatedFileName: String = target.generatedFile.name 33 | val generatingRuleName: String = target.generatedFile.generatingRule 34 | override val name: String 35 | get() = generatedFileName 36 | } 37 | 38 | val type: BazelTargetType 39 | get() = 40 | when (target.type) { 41 | Build.Target.Discriminator.RULE -> BazelTargetType.RULE 42 | Build.Target.Discriminator.SOURCE_FILE -> BazelTargetType.SOURCE_FILE 43 | Build.Target.Discriminator.GENERATED_FILE -> BazelTargetType.GENERATED_FILE 44 | Build.Target.Discriminator.PACKAGE_GROUP -> BazelTargetType.PACKAGE_GROUP 45 | Build.Target.Discriminator.ENVIRONMENT_GROUP -> BazelTargetType.ENVIRONMENT_GROUP 46 | else -> BazelTargetType.UNKNOWN 47 | } 48 | 49 | abstract val name: String 50 | } 51 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/bazel/BazelTargetType.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.bazel 2 | 3 | enum class BazelTargetType { 4 | RULE, 5 | SOURCE_FILE, 6 | GENERATED_FILE, 7 | PACKAGE_GROUP, 8 | ENVIRONMENT_GROUP, 9 | UNKNOWN 10 | } 11 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/cli/BazelDiff.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.cli 2 | 3 | import picocli.CommandLine 4 | import picocli.CommandLine.Model.CommandSpec 5 | import picocli.CommandLine.Spec 6 | 7 | @CommandLine.Command( 8 | name = "bazel-diff", 9 | description = ["Writes to a file the impacted targets between two Bazel graph JSON files"], 10 | subcommands = [GenerateHashesCommand::class, GetImpactedTargetsCommand::class], 11 | mixinStandardHelpOptions = true, 12 | versionProvider = VersionProvider::class, 13 | ) 14 | class BazelDiff : Runnable { 15 | @CommandLine.Option( 16 | names = ["-v", "--verbose"], 17 | description = ["Display query string, missing files and elapsed time"], 18 | scope = CommandLine.ScopeType.INHERIT, 19 | ) 20 | var verbose = false 21 | 22 | @CommandLine.Option( 23 | names = ["--debug"], 24 | hidden = true, 25 | scope = CommandLine.ScopeType.INHERIT, 26 | defaultValue = "\${sys:DEBUG:-false}") 27 | var debug = false 28 | 29 | @Spec lateinit var spec: CommandSpec 30 | 31 | override fun run() { 32 | throw CommandLine.ParameterException(spec.commandLine(), "Missing required subcommand") 33 | } 34 | 35 | fun isVerbose(): Boolean { 36 | return verbose || debug 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/cli/GenerateHashesCommand.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.cli 2 | 3 | import com.bazel_diff.cli.converter.CommaSeparatedValueConverter 4 | import com.bazel_diff.cli.converter.NormalisingPathConverter 5 | import com.bazel_diff.cli.converter.OptionsConverter 6 | import com.bazel_diff.di.hasherModule 7 | import com.bazel_diff.di.loggingModule 8 | import com.bazel_diff.di.serialisationModule 9 | import com.bazel_diff.interactor.GenerateHashesInteractor 10 | import java.io.File 11 | import java.nio.file.Path 12 | import java.util.concurrent.Callable 13 | import org.koin.core.context.startKoin 14 | import org.koin.core.context.stopKoin 15 | import picocli.CommandLine 16 | 17 | @CommandLine.Command( 18 | name = "generate-hashes", 19 | mixinStandardHelpOptions = true, 20 | description = 21 | ["Writes to a file the SHA256 hashes for each Bazel Target in the provided workspace."], 22 | versionProvider = VersionProvider::class) 23 | class GenerateHashesCommand : Callable { 24 | @CommandLine.ParentCommand private lateinit var parent: BazelDiff 25 | 26 | @CommandLine.Option( 27 | names = ["-w", "--workspacePath"], 28 | description = ["Path to Bazel workspace directory."], 29 | scope = CommandLine.ScopeType.INHERIT, 30 | required = true, 31 | converter = [NormalisingPathConverter::class]) 32 | lateinit var workspacePath: Path 33 | 34 | @CommandLine.Option( 35 | names = ["-b", "--bazelPath"], 36 | description = 37 | [ 38 | "Path to Bazel binary. If not specified, the Bazel binary available in PATH will be used."], 39 | scope = CommandLine.ScopeType.INHERIT, 40 | defaultValue = "bazel", 41 | ) 42 | lateinit var bazelPath: Path 43 | 44 | @CommandLine.Option( 45 | names = ["--contentHashPath"], 46 | description = 47 | [ 48 | "Path to content hash json file. It's a map which maps relative file path from workspace path to its content hash. Files in this map will skip content hashing and use provided value"], 49 | scope = CommandLine.ScopeType.INHERIT, 50 | required = false) 51 | var contentHashPath: File? = null 52 | 53 | @CommandLine.Option( 54 | names = ["-so", "--bazelStartupOptions"], 55 | description = 56 | ["Additional space separated Bazel client startup options used when invoking Bazel"], 57 | scope = CommandLine.ScopeType.INHERIT, 58 | converter = [OptionsConverter::class], 59 | ) 60 | var bazelStartupOptions: List = emptyList() 61 | 62 | @CommandLine.Option( 63 | names = ["-co", "--bazelCommandOptions"], 64 | description = 65 | ["Additional space separated Bazel command options used when invoking `bazel query`"], 66 | scope = CommandLine.ScopeType.INHERIT, 67 | converter = [OptionsConverter::class], 68 | ) 69 | var bazelCommandOptions: List = emptyList() 70 | 71 | @CommandLine.Option( 72 | names = ["--fineGrainedHashExternalRepos"], 73 | description = 74 | [ 75 | "Comma separate list of external repos in which fine-grained hashes are computed for the targets. By default, external repos are treated as an opaque blob. If an external repo is specified here, bazel-diff instead computes the hash for individual targets. For example, one wants to specify `maven` here if they user rules_jvm_external so that individual third party dependency change won't invalidate all targets in the mono repo."], 76 | scope = CommandLine.ScopeType.INHERIT, 77 | converter = [CommaSeparatedValueConverter::class], 78 | ) 79 | var fineGrainedHashExternalRepos: Set = emptySet() 80 | 81 | @CommandLine.Option( 82 | names = ["--useCquery"], 83 | negatable = true, 84 | description = 85 | [ 86 | "If true, use cquery instead of query when generating dependency graphs. Using cquery would yield more accurate build graph at the cost of slower query execution. When this is set, one usually also wants to set `--cqueryCommandOptions` to specify a targeting platform. Note that this flag only works with Bazel 6.2.0 or above because lower versions does not support `--query_file` flag."], 87 | scope = CommandLine.ScopeType.INHERIT) 88 | var useCquery = false 89 | 90 | @CommandLine.Option( 91 | names = ["--includeTargetType"], 92 | negatable = true, 93 | description = 94 | [ 95 | "Whether include target type in the generated JSON or not.\n" + 96 | "If false, the generate JSON schema is: {\"\": \"\"}\n" + 97 | "If true, the generate JSON schema is: {\"\": \"#\" }"], 98 | scope = CommandLine.ScopeType.INHERIT) 99 | var includeTargetType = false 100 | 101 | @CommandLine.Option( 102 | names = ["-tt", "--targetType"], 103 | split = ",", 104 | scope = CommandLine.ScopeType.LOCAL, 105 | description = 106 | [ 107 | "The types of targets to filter. Use comma (,) to separate multiple values, e.g. '--targetType=SourceFile,Rule,GeneratedFile'."]) 108 | var targetType: Set? = null 109 | 110 | @CommandLine.Option( 111 | names = ["--cqueryCommandOptions"], 112 | description = 113 | [ 114 | "Additional space separated Bazel command options used when invoking `bazel cquery`. This flag is has no effect if `--useCquery`is false."], 115 | scope = CommandLine.ScopeType.INHERIT, 116 | converter = [OptionsConverter::class], 117 | ) 118 | var cqueryCommandOptions: List = emptyList() 119 | 120 | @CommandLine.Option( 121 | names = ["-k", "--keep_going"], 122 | negatable = true, 123 | description = 124 | [ 125 | "This flag controls if `bazel query` will be executed with the `--keep_going` flag or not. Disabling this flag allows you to catch configuration issues in your Bazel graph, but may not work for some Bazel setups. Defaults to `true`"], 126 | scope = CommandLine.ScopeType.INHERIT) 127 | var keepGoing = true 128 | 129 | @CommandLine.Option( 130 | names = ["-s", "--seed-filepaths"], 131 | description = 132 | [ 133 | "A text file containing a newline separated list of filepaths, each of these filepaths will be read and used as a seed for all targets."]) 134 | var seedFilepaths: File? = null 135 | 136 | @CommandLine.Parameters( 137 | index = "0", 138 | description = 139 | [ 140 | "The filepath to write the resulting JSON of dictionary target => SHA-256 values. If not specified, the JSON will be written to STDOUT."], 141 | defaultValue = CommandLine.Parameters.NULL_VALUE) 142 | var outputPath: File? = null 143 | 144 | @CommandLine.Option( 145 | names = ["--ignoredRuleHashingAttributes"], 146 | description = ["Attributes that should be ignored when hashing rule targets."], 147 | scope = CommandLine.ScopeType.INHERIT, 148 | converter = [CommaSeparatedValueConverter::class], 149 | ) 150 | var ignoredRuleHashingAttributes: Set = emptySet() 151 | 152 | @CommandLine.Option( 153 | names = ["-d", "--depEdgesFile"], 154 | description = 155 | [ 156 | "Path to the file where dependency edges are written to. If not specified, the dependency edges will not be written to a file. Needed for computing build graph distance metrics. See bazel-diff docs for more details about build graph distance metrics."], 157 | scope = CommandLine.ScopeType.INHERIT, 158 | defaultValue = CommandLine.Parameters.NULL_VALUE) 159 | var depsMappingJSONPath: File? = null 160 | 161 | @CommandLine.Option( 162 | names = ["-m", "--modified-filepaths"], 163 | description = 164 | [ 165 | "Experimental: A text file containing a newline separated list of filepaths (relative to the workspace) these filepaths should represent the modified files between the specified revisions and will be used to scope what files are hashed during hash generation."]) 166 | var modifiedFilepaths: File? = null 167 | 168 | @CommandLine.Option( 169 | names = ["--excludeExternalTargets"], 170 | negatable = true, 171 | description = 172 | [ 173 | "If true, exclude external targets. This must be switched on when using `--noenable_workspace` Bazel command line option. Defaults to `false`."], 174 | scope = CommandLine.ScopeType.INHERIT) 175 | var excludeExternalTargets = false 176 | 177 | @CommandLine.Spec lateinit var spec: CommandLine.Model.CommandSpec 178 | 179 | override fun call(): Int { 180 | validate(contentHashPath = contentHashPath) 181 | 182 | startKoin { 183 | modules( 184 | hasherModule( 185 | workspacePath, 186 | bazelPath, 187 | contentHashPath, 188 | bazelStartupOptions, 189 | bazelCommandOptions, 190 | cqueryCommandOptions, 191 | useCquery, 192 | keepGoing, 193 | depsMappingJSONPath != null, 194 | fineGrainedHashExternalRepos, 195 | excludeExternalTargets, 196 | ), 197 | loggingModule(parent.verbose), 198 | serialisationModule(), 199 | ) 200 | } 201 | 202 | return when (GenerateHashesInteractor() 203 | .execute( 204 | seedFilepaths, 205 | outputPath, 206 | depsMappingJSONPath, 207 | ignoredRuleHashingAttributes, 208 | targetType, 209 | includeTargetType, 210 | modifiedFilepaths)) { 211 | true -> CommandLine.ExitCode.OK 212 | false -> CommandLine.ExitCode.SOFTWARE 213 | }.also { stopKoin() } 214 | } 215 | 216 | private fun validate(contentHashPath: File?) { 217 | contentHashPath?.let { 218 | if (!it.canRead()) { 219 | throw CommandLine.ParameterException( 220 | spec.commandLine(), 221 | "Incorrect contentHashFilePath: file doesn't exist or can't be read.") 222 | } 223 | } 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/cli/GetImpactedTargetsCommand.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.cli 2 | 3 | import com.bazel_diff.di.loggingModule 4 | import com.bazel_diff.di.serialisationModule 5 | import com.bazel_diff.interactor.CalculateImpactedTargetsInteractor 6 | import com.bazel_diff.interactor.DeserialiseHashesInteractor 7 | import java.io.BufferedWriter 8 | import java.io.File 9 | import java.io.FileDescriptor 10 | import java.io.FileWriter 11 | import java.io.IOException 12 | import java.util.concurrent.Callable 13 | import org.koin.core.context.startKoin 14 | import org.koin.core.context.stopKoin 15 | import picocli.CommandLine 16 | 17 | @CommandLine.Command( 18 | name = "get-impacted-targets", 19 | description = ["Command-line utility to analyze the state of the bazel build graph"], 20 | ) 21 | class GetImpactedTargetsCommand : Callable { 22 | @CommandLine.ParentCommand private lateinit var parent: BazelDiff 23 | 24 | @CommandLine.Option( 25 | names = ["-sh", "--startingHashes"], 26 | scope = CommandLine.ScopeType.LOCAL, 27 | description = 28 | [ 29 | "The path to the JSON file of target hashes for the initial revision. Run 'generate-hashes' to get this value."], 30 | required = true, 31 | ) 32 | lateinit var startingHashesJSONPath: File 33 | 34 | @CommandLine.Option( 35 | names = ["-fh", "--finalHashes"], 36 | scope = CommandLine.ScopeType.LOCAL, 37 | description = 38 | [ 39 | "The path to the JSON file of target hashes for the final revision. Run 'generate-hashes' to get this value."], 40 | required = true, 41 | ) 42 | lateinit var finalHashesJSONPath: File 43 | 44 | @CommandLine.Option( 45 | names = ["-d", "--depEdgesFile"], 46 | description = 47 | [ 48 | "Path to the file where dependency edges are. If specified, build graph distance metrics will be computed from the given hash data."], 49 | scope = CommandLine.ScopeType.INHERIT, 50 | defaultValue = CommandLine.Parameters.NULL_VALUE) 51 | var depsMappingJSONPath: File? = null 52 | 53 | @CommandLine.Option( 54 | names = ["-tt", "--targetType"], 55 | split = ",", 56 | scope = CommandLine.ScopeType.LOCAL, 57 | description = 58 | [ 59 | "The types of targets to filter. Use comma (,) to separate multiple values, e.g. '--targetType=SourceFile,Rule,GeneratedFile'."]) 60 | var targetType: Set? = null 61 | 62 | @CommandLine.Option( 63 | names = ["-o", "--output"], 64 | scope = CommandLine.ScopeType.LOCAL, 65 | description = 66 | [ 67 | "Filepath to write the impacted Bazel targets to. If using depEdgesFile: formatted in json, otherwise: newline separated. If not specified, the output will be written to STDOUT."], 68 | ) 69 | var outputPath: File? = null 70 | 71 | @CommandLine.Spec lateinit var spec: CommandLine.Model.CommandSpec 72 | 73 | override fun call(): Int { 74 | startKoin { modules(serialisationModule(), loggingModule(parent.verbose)) } 75 | 76 | validate() 77 | val deserialiser = DeserialiseHashesInteractor() 78 | val from = deserialiser.executeTargetHash(startingHashesJSONPath) 79 | val to = deserialiser.executeTargetHash(finalHashesJSONPath) 80 | 81 | val outputWriter = 82 | BufferedWriter( 83 | when (val path = outputPath) { 84 | null -> FileWriter(FileDescriptor.out) 85 | else -> FileWriter(path) 86 | }) 87 | 88 | return try { 89 | if (depsMappingJSONPath != null) { 90 | val depsMapping = deserialiser.deserializeDeps(depsMappingJSONPath!!) 91 | CalculateImpactedTargetsInteractor() 92 | .executeWithDistances(from, to, depsMapping, outputWriter, targetType) 93 | } else { 94 | CalculateImpactedTargetsInteractor().execute(from, to, outputWriter, targetType) 95 | } 96 | CommandLine.ExitCode.OK 97 | } catch (e: IOException) { 98 | CommandLine.ExitCode.SOFTWARE 99 | } 100 | .also { stopKoin() } 101 | } 102 | 103 | private fun validate() { 104 | if (!startingHashesJSONPath.canRead()) { 105 | throw CommandLine.ParameterException( 106 | spec.commandLine(), "Incorrect starting hashes: file doesn't exist or can't be read.") 107 | } 108 | if (!finalHashesJSONPath.canRead()) { 109 | throw CommandLine.ParameterException( 110 | spec.commandLine(), "Incorrect final hashes: file doesn't exist or can't be read.") 111 | } 112 | if (depsMappingJSONPath != null && !depsMappingJSONPath!!.canRead()) { 113 | throw CommandLine.ParameterException( 114 | spec.commandLine(), "Incorrect dep edges file: file doesn't exist or can't be read.") 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/cli/VersionProvider.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.cli 2 | 3 | import java.io.BufferedReader 4 | import java.io.InputStreamReader 5 | import picocli.CommandLine.IVersionProvider 6 | 7 | class VersionProvider : IVersionProvider { 8 | override fun getVersion(): Array { 9 | val classLoader = this::class.java.classLoader 10 | val inputStream = 11 | classLoader.getResourceAsStream("cli/version") 12 | ?: classLoader.getResourceAsStream("version") 13 | ?: throw IllegalArgumentException( 14 | "unknown version as version file not found in resources") 15 | 16 | val version = BufferedReader(InputStreamReader(inputStream)).use { it.readText().trim() } 17 | return arrayOf(version) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/cli/converter/CommaSeparatedValueConverter.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.cli.converter 2 | 3 | import picocli.CommandLine.ITypeConverter 4 | 5 | class CommaSeparatedValueConverter : ITypeConverter> { 6 | override fun convert(value: String): Set { 7 | return value.split(",").dropLastWhile { it.isEmpty() }.toSet() 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/cli/converter/NormalisingPathConverter.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.cli.converter 2 | 3 | import java.io.File 4 | import java.nio.file.Path 5 | import picocli.CommandLine.ITypeConverter 6 | 7 | class NormalisingPathConverter : ITypeConverter { 8 | override fun convert(value: String): Path = File(value).toPath().normalize() 9 | } 10 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/cli/converter/OptionsConverter.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.cli.converter 2 | 3 | import picocli.CommandLine.ITypeConverter 4 | 5 | class OptionsConverter : ITypeConverter> { 6 | override fun convert(value: String): List { 7 | return value.split(" ").dropLastWhile { it.isEmpty() } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/di/Modules.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.di 2 | 3 | import com.bazel_diff.bazel.BazelClient 4 | import com.bazel_diff.bazel.BazelQueryService 5 | import com.bazel_diff.hash.* 6 | import com.bazel_diff.io.ContentHashProvider 7 | import com.bazel_diff.log.Logger 8 | import com.bazel_diff.log.StderrLogger 9 | import com.bazel_diff.process.Redirect 10 | import com.bazel_diff.process.process 11 | import com.google.gson.GsonBuilder 12 | import java.io.File 13 | import java.nio.file.Path 14 | import java.nio.file.Paths 15 | import kotlinx.coroutines.ExperimentalCoroutinesApi 16 | import kotlinx.coroutines.runBlocking 17 | import org.koin.core.module.Module 18 | import org.koin.core.qualifier.named 19 | import org.koin.dsl.module 20 | 21 | @OptIn(ExperimentalCoroutinesApi::class) 22 | fun hasherModule( 23 | workingDirectory: Path, 24 | bazelPath: Path, 25 | contentHashPath: File?, 26 | startupOptions: List, 27 | commandOptions: List, 28 | cqueryOptions: List, 29 | useCquery: Boolean, 30 | keepGoing: Boolean, 31 | trackDeps: Boolean, 32 | fineGrainedHashExternalRepos: Set, 33 | excludeExternalTargets: Boolean, 34 | ): Module = module { 35 | val cmd: MutableList = 36 | ArrayList().apply { 37 | add(bazelPath.toString()) 38 | addAll(startupOptions) 39 | add("info") 40 | add("output_base") 41 | } 42 | val result = runBlocking { 43 | process( 44 | *cmd.toTypedArray(), 45 | stdout = Redirect.CAPTURE, 46 | workingDirectory = workingDirectory.toFile(), 47 | stderr = Redirect.PRINT, 48 | destroyForcibly = true, 49 | ) 50 | } 51 | val outputPath = Paths.get(result.output.single()) 52 | val debug = System.getProperty("DEBUG", "false").equals("true") 53 | single { 54 | BazelQueryService( 55 | workingDirectory, 56 | bazelPath, 57 | startupOptions, 58 | commandOptions, 59 | cqueryOptions, 60 | keepGoing, 61 | debug) 62 | } 63 | single { BazelClient(useCquery, fineGrainedHashExternalRepos, excludeExternalTargets) } 64 | single { BuildGraphHasher(get()) } 65 | single { TargetHasher() } 66 | single { RuleHasher(useCquery, trackDeps, fineGrainedHashExternalRepos) } 67 | single { SourceFileHasherImpl(fineGrainedHashExternalRepos) } 68 | single { ExternalRepoResolver(workingDirectory, bazelPath, outputPath) } 69 | single(named("working-directory")) { workingDirectory } 70 | single(named("output-base")) { outputPath } 71 | single { ContentHashProvider(contentHashPath) } 72 | } 73 | 74 | fun loggingModule(verbose: Boolean) = module { single { StderrLogger(verbose) } } 75 | 76 | fun serialisationModule() = module { 77 | single { GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create() } 78 | } 79 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/extensions/ByteArray.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.extensions 2 | 3 | import java.nio.charset.StandardCharsets 4 | 5 | fun ByteArray.toHexString(): String { 6 | val hexChars = ByteArray(size * 2) 7 | for (j in indices) { 8 | val v = get(j).toInt() and 0xFF 9 | hexChars[j * 2] = HEX_ARRAY[v ushr 4] 10 | hexChars[j * 2 + 1] = HEX_ARRAY[v and 0x0F] 11 | } 12 | return String(hexChars, StandardCharsets.UTF_8) 13 | } 14 | 15 | private val HEX_ARRAY = "0123456789abcdef".toByteArray(StandardCharsets.US_ASCII) 16 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/extensions/ByteBuffer.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.extensions 2 | 3 | import java.nio.Buffer 4 | import java.nio.ByteBuffer 5 | 6 | /** Fix for NoSuchMethodError for JRE8 */ 7 | fun ByteBuffer.compatClear() = ((this as Buffer).clear() as ByteBuffer) 8 | 9 | fun ByteBuffer.compatFlip() = ((this as Buffer).flip() as ByteBuffer) 10 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/extensions/HashingExtensions.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.hash 2 | 3 | import com.bazel_diff.extensions.compatClear 4 | import com.bazel_diff.extensions.compatFlip 5 | import com.bazel_diff.io.ByteBufferPool 6 | import com.google.common.hash.Hasher 7 | import com.google.common.hash.Hashing 8 | import java.io.BufferedInputStream 9 | import java.io.File 10 | import java.io.FileInputStream 11 | 12 | fun sha256(block: Hasher.() -> Unit): ByteArray { 13 | val hasher = Hashing.sha256().newHasher() 14 | hasher.apply(block) 15 | return hasher.hash().asBytes().clone() 16 | } 17 | 18 | fun Hasher.safePutBytes(block: ByteArray?) = block?.let { putBytes(it) } 19 | 20 | fun Hasher.putFile(file: File) { 21 | BufferedInputStream(FileInputStream(file.absolutePath.toString())).use { stream -> 22 | val buffer = pool.borrow() 23 | val array = buffer!!.array() // Available for non-direct buffers 24 | while (true) { 25 | var length: Int 26 | if (stream.read(array).also { length = it } == -1) break 27 | buffer.compatFlip() 28 | putBytes(array, 0, length) 29 | buffer.compatClear() 30 | } 31 | pool.recycle(buffer) 32 | } 33 | } 34 | 35 | private val pool = ByteBufferPool(1024, 10240) // 10kb 36 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/hash/BuildGraphHasher.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.hash 2 | 3 | import com.bazel_diff.bazel.BazelClient 4 | import com.bazel_diff.bazel.BazelRule 5 | import com.bazel_diff.bazel.BazelSourceFileTarget 6 | import com.bazel_diff.bazel.BazelTarget 7 | import com.bazel_diff.extensions.toHexString 8 | import com.bazel_diff.log.Logger 9 | import com.google.common.collect.Sets 10 | import java.nio.file.Path 11 | import java.util.Calendar 12 | import java.util.concurrent.ConcurrentHashMap 13 | import java.util.concurrent.ConcurrentMap 14 | import java.util.concurrent.atomic.AtomicReference 15 | import java.util.stream.Collectors 16 | import kotlin.io.path.readBytes 17 | import kotlinx.coroutines.Dispatchers 18 | import kotlinx.coroutines.async 19 | import kotlinx.coroutines.runBlocking 20 | import org.koin.core.component.KoinComponent 21 | import org.koin.core.component.inject 22 | 23 | class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent { 24 | private val targetHasher: TargetHasher by inject() 25 | private val sourceFileHasher: SourceFileHasher by inject() 26 | private val logger: Logger by inject() 27 | 28 | fun hashAllBazelTargetsAndSourcefiles( 29 | seedFilepaths: Set = emptySet(), 30 | ignoredAttrs: Set = emptySet(), 31 | modifiedFilepaths: Set = emptySet() 32 | ): Map { 33 | val (sourceDigests, allTargets) = 34 | runBlocking { 35 | val targetsTask = async(Dispatchers.IO) { bazelClient.queryAllTargets() } 36 | val allTargets = targetsTask.await() 37 | val sourceTargets = 38 | allTargets 39 | .filter { it is BazelTarget.SourceFile } 40 | .map { it as BazelTarget.SourceFile } 41 | 42 | val sourceDigestsFuture = 43 | async(Dispatchers.IO) { 44 | val sourceHashDurationEpoch = Calendar.getInstance().getTimeInMillis() 45 | val sourceFileTargets = hashSourcefiles(sourceTargets, modifiedFilepaths) 46 | val sourceHashDuration = 47 | Calendar.getInstance().getTimeInMillis() - sourceHashDurationEpoch 48 | logger.i { "Source file hashes calculated in $sourceHashDuration" } 49 | sourceFileTargets 50 | } 51 | 52 | Pair(sourceDigestsFuture.await(), allTargets) 53 | } 54 | val seedForFilepaths = createSeedForFilepaths(seedFilepaths) 55 | return hashAllTargets( 56 | seedForFilepaths, sourceDigests, allTargets, ignoredAttrs, modifiedFilepaths) 57 | } 58 | 59 | private fun hashSourcefiles( 60 | targets: List, 61 | modifiedFilepaths: Set 62 | ): ConcurrentMap { 63 | val exception = AtomicReference(null) 64 | val result: ConcurrentMap = 65 | targets 66 | .parallelStream() 67 | .map { sourceFile: BazelTarget.SourceFile -> 68 | val seed = sha256 { 69 | safePutBytes(sourceFile.name.toByteArray()) 70 | for (subinclude in sourceFile.subincludeList) { 71 | safePutBytes(subinclude.toByteArray()) 72 | } 73 | } 74 | try { 75 | val sourceFileTarget = BazelSourceFileTarget(sourceFile.name, seed) 76 | Pair( 77 | sourceFileTarget.name, 78 | sourceFileHasher.digest(sourceFileTarget, modifiedFilepaths)) 79 | } catch (e: Exception) { 80 | exception.set(e) 81 | null 82 | } 83 | } 84 | .filter { pair -> pair != null } 85 | .collect( 86 | Collectors.toConcurrentMap( 87 | { pair -> pair!!.first }, 88 | { pair -> pair!!.second }, 89 | )) 90 | 91 | exception.get()?.let { throw it } 92 | return result 93 | } 94 | 95 | private fun hashAllTargets( 96 | seedHash: ByteArray, 97 | sourceDigests: ConcurrentMap, 98 | allTargets: List, 99 | ignoredAttrs: Set, 100 | modifiedFilepaths: Set 101 | ): Map { 102 | val ruleHashes: ConcurrentMap = ConcurrentHashMap() 103 | val targetToRule: MutableMap = HashMap() 104 | traverseGraph(allTargets, targetToRule) 105 | 106 | return allTargets 107 | .parallelStream() 108 | .map { target: BazelTarget -> 109 | val targetDigest = 110 | targetHasher.digest( 111 | target, 112 | targetToRule, 113 | sourceDigests, 114 | ruleHashes, 115 | seedHash, 116 | ignoredAttrs, 117 | modifiedFilepaths) 118 | Pair( 119 | target.name, 120 | TargetHash( 121 | target.javaClass.name.substringAfterLast('$'), 122 | targetDigest.overallDigest.toHexString(), 123 | targetDigest.directDigest.toHexString(), 124 | targetDigest.deps, 125 | )) 126 | } 127 | .filter { targetEntry: Pair? -> targetEntry != null } 128 | .collect( 129 | Collectors.toMap( 130 | { obj: Pair -> obj.first }, 131 | { obj: Pair -> obj.second }, 132 | )) 133 | } 134 | 135 | /** Traverses the list of targets and revisits the targets with yet-unknown generating rule */ 136 | private fun traverseGraph( 137 | allTargets: List, 138 | targetToRule: MutableMap 139 | ) { 140 | var targetsToAnalyse: Set = Sets.newHashSet(allTargets) 141 | while (!targetsToAnalyse.isEmpty()) { 142 | val initialSize = targetsToAnalyse.size 143 | val nextTargets: MutableSet = Sets.newHashSet() 144 | for (target in targetsToAnalyse) { 145 | val targetName = target.name 146 | when (target) { 147 | is BazelTarget.GeneratedFile -> { 148 | targetToRule[target.generatingRuleName]?.let { targetToRule[targetName] = it } 149 | ?: nextTargets.add(target) 150 | } 151 | is BazelTarget.Rule -> targetToRule[targetName] = target.rule 152 | is BazelTarget.SourceFile -> continue 153 | } 154 | } 155 | val newSize = nextTargets.size 156 | if (newSize >= initialSize) { 157 | throw RuntimeException("Not possible to traverse the build graph") 158 | } 159 | targetsToAnalyse = nextTargets 160 | } 161 | } 162 | 163 | private fun createSeedForFilepaths(seedFilepaths: Set): ByteArray { 164 | if (seedFilepaths.isEmpty()) { 165 | return ByteArray(0) 166 | } 167 | return sha256 { 168 | for (path in seedFilepaths) { 169 | putBytes(path.readBytes()) 170 | } 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/bazel_diff/hash/ExternalRepoResolver.kt: -------------------------------------------------------------------------------- 1 | package com.bazel_diff.hash 2 | 3 | import com.bazel_diff.log.Logger 4 | import com.google.common.cache.CacheBuilder 5 | import com.google.common.cache.CacheLoader 6 | import java.nio.file.Files 7 | import java.nio.file.Path 8 | import java.nio.file.Paths 9 | import org.koin.core.component.KoinComponent 10 | import org.koin.core.component.inject 11 | 12 | class ExternalRepoResolver( 13 | private val workingDirectory: Path, 14 | private val bazelPath: Path, 15 | private val outputBase: Path, 16 | ) : KoinComponent { 17 | private val logger: Logger by inject() 18 | 19 | private val externalRoot: Path by lazy { outputBase.resolve("external") } 20 | 21 | private val cache = 22 | CacheBuilder.newBuilder() 23 | .build( 24 | CacheLoader.from { repoName: String -> 25 | val externalRepoRoot = externalRoot.resolve(repoName) 26 | if (Files.exists(externalRepoRoot)) { 27 | return@from externalRepoRoot 28 | } 29 | resolveBzlModPath(repoName) 30 | }) 31 | 32 | fun resolveExternalRepoRoot(repoName: String): Path { 33 | return cache.get(repoName) 34 | } 35 | 36 | private fun resolveBzlModPath(repoName: String): Path { 37 | // Query result line should look something like "/external//some/bazel/target: