├── .bazelci
├── postsubmit.yml
└── presubmit.yml
├── .bazelrc
├── .bazelversion
├── .gitignore
├── BUILD
├── CONTRIBUTING.md
├── ISSUE_TEMPLATE.md
├── LICENSE
├── MODULE.bazel
├── PULL_REQUEST_TEMPLATE.md
├── README.md
├── WORKSPACE
├── WORKSPACE.bzlmod
├── bb-icon.png
├── benchmark.py
├── benchmark_test.py
├── report
├── BUILD
├── generate_master_report.py
└── generate_report.py
├── testutils
├── BUILD
├── __init__.py
└── fakes.py
├── third_party
├── BUILD
├── requirements.in
└── requirements.txt
└── utils
├── BUILD
├── __init__.py
├── bazel.py
├── bazel_test.py
├── benchmark_config.py
├── benchmark_config_test.py
├── bigquery_upload.py
├── json_profiles_merger.py
├── json_profiles_merger_lib.py
├── json_profiles_merger_lib_test.py
├── logger.py
├── output_handling.py
├── storage_upload.py
├── values.py
└── values_test.py
/.bazelci/postsubmit.yml:
--------------------------------------------------------------------------------
1 | ---
2 | tasks:
3 | ubuntu2204:
4 | include_json_profile:
5 | - build
6 | - test
7 | build_targets:
8 | - "//..."
9 | test_targets:
10 | - "//..."
11 | macos_arm64:
12 | include_json_profile:
13 | - build
14 | - test
15 | build_targets:
16 | - "//..."
17 | test_targets:
18 | - "//..."
19 |
--------------------------------------------------------------------------------
/.bazelci/presubmit.yml:
--------------------------------------------------------------------------------
1 | ---
2 | tasks:
3 | ubuntu2204:
4 | build_targets:
5 | - "//..."
6 | test_targets:
7 | - "//..."
8 | macos_arm64:
9 | build_targets:
10 | - "//..."
11 | test_targets:
12 | - "//..."
13 |
--------------------------------------------------------------------------------
/.bazelrc:
--------------------------------------------------------------------------------
1 | build --incompatible_disallow_empty_glob
2 |
--------------------------------------------------------------------------------
/.bazelversion:
--------------------------------------------------------------------------------
1 | 7.3.1
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Ignore backup files.
2 | *~
3 | # Ignore Vim swap files.
4 | .*.swp
5 | # Ignore files generated by IDEs.
6 | /.classpath
7 | /.factorypath
8 | /.idea/
9 | /.ijwb/
10 | /.project
11 | /.settings
12 | /.vscode/
13 |
14 | # Byte-compiled / optimized / DLL files
15 | __pycache__/
16 | *.py[cod]
17 | *$py.class
18 |
19 | # Config file
20 | utils/config.py
21 |
22 | # Bazel
23 | bazel-*
24 | MODULE.bazel.lock
25 |
26 | # Python venv
27 | .venv/
--------------------------------------------------------------------------------
/BUILD:
--------------------------------------------------------------------------------
1 | load("@rules_python//python:defs.bzl", "py_binary", "py_test")
2 | load("@third_party//:requirements.bzl", "requirement")
3 |
4 | # TODO(https://github.com/bazelbuild/bazel-bench/issues/36): Make these work for python3.
5 | py_binary(
6 | name = "benchmark",
7 | srcs = ["benchmark.py"],
8 | deps = [
9 | "//utils",
10 | requirement("absl-py"),
11 | requirement("GitPython"),
12 | requirement("gitdb2"),
13 | ],
14 | )
15 |
16 | py_test(
17 | name = "benchmark_test",
18 | srcs = ["benchmark_test.py"],
19 | deps = [
20 | ":benchmark",
21 | "//testutils",
22 | requirement("mock"),
23 | ],
24 | )
25 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement. You (or your employer) retain the copyright to your contribution;
10 | this simply gives us permission to use and redistribute your contributions as
11 | part of the project. Head over to to see
12 | your current agreements on file or to sign a new one.
13 |
14 | You generally only need to submit a CLA once, so if you've already submitted one
15 | (even if it was for a different project), you probably don't need to do it
16 | again.
17 |
18 | ## Code reviews
19 |
20 | All submissions, including submissions by project members, require review. We
21 | use GitHub pull requests for this purpose. Consult
22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
23 | information on using pull requests.
24 |
25 | ## Community Guidelines
26 |
27 | This project follows [Google's Open Source Community
28 | Guidelines](https://opensource.google.com/conduct/).
29 |
--------------------------------------------------------------------------------
/ISSUE_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | **Description of the problem / feature request:**
2 |
3 | > Replace this line with your answer.
4 |
5 | **Feature requests: what underlying problem are you trying to solve with this feature?**
6 |
7 | > Replace this line with your answer.
8 |
9 | **Bugs: what's the simplest, easiest way to reproduce this bug? Please provide a minimal example if possible.**
10 |
11 | > Replace this line with your answer.
12 |
13 | **Any other information, logs, or outputs that you want to share?**
14 |
15 | > Replace these lines with your answer.
16 | >
17 | > If the files are large, upload as attachment or provide link.
18 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/MODULE.bazel:
--------------------------------------------------------------------------------
1 | module(
2 | name = "bazel-bench",
3 | version = "0.0.0",
4 | )
5 |
6 | bazel_dep(
7 | name = "rules_python",
8 | version = "0.35.0",
9 | )
10 |
11 | # -- bazel_dep definitions -- #
12 | python = use_extension(
13 | "@rules_python//python/extensions:python.bzl",
14 | "python",
15 | dev_dependency = True,
16 | )
17 | python.toolchain(
18 | python_version = "3.10",
19 | )
20 |
21 | pip = use_extension(
22 | "@rules_python//python/extensions:pip.bzl",
23 | "pip",
24 | dev_dependency = True,
25 | )
26 | pip.parse(
27 | hub_name = "third_party",
28 | python_version = "3.10",
29 | requirements_lock = "//third_party:requirements.txt",
30 | )
31 | use_repo(pip, "third_party")
32 |
--------------------------------------------------------------------------------
/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | **What this PR does and why we need it:**
2 |
3 | > Replace this line with your answer.
4 |
5 | **New changes / Issues that this PR fixes:**
6 |
7 | > Replace this line with your answer.
8 |
9 | **Special notes for reviewer:**
10 |
11 | > Replace this line with your answer.
12 |
13 | **Does this require a change in the script's interface or the BigQuery's table structure?**
14 |
15 | > Replace this line with your answer.
16 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Bazel Performance Benchmarking
2 |
3 | [](https://buildkite.com/bazel/bazel-bench)
4 |
5 | **Status**: WIP
6 |
7 | 
8 |
9 | # Setup
10 |
11 | Pre-requisites: `git` and `bazel`.
12 |
13 | ```
14 | # Clone bazel-bench.
15 | $ git clone https://github.com/bazelbuild/bazel-bench.git
16 | $ cd bazel-bench
17 | ```
18 |
19 | To do a test run, run the following command (if you're on Windows, populate
20 | `--data_directory` with an appropriate Windows-style path):
21 |
22 | ```shell
23 | $ bazel run :benchmark \
24 | -- \
25 | --bazel_commits=b8468a6b68a405e1a5767894426d3ea9a1a2f22f,ad503849e78b98d762f03168de5a336904280150 \
26 | --project_source=https://github.com/bazelbuild/rules_cc.git \
27 | --data_directory=/tmp/bazel-bench-data \
28 | --verbose \
29 | -- build //:all
30 | ```
31 |
32 | The Bazel commits might be too old and no longer buildable by your local Bazel. Replace them with the more recent commits from [bazelbuild/bazel](https://github.com/bazelbuild/bazel). The above command would print a result table on the terminal and outputs a csv
33 | file to the specified `--data_directory`.
34 |
35 | ## Syntax
36 |
37 | Bazel-bench has the following syntax:
38 |
39 | ```shell
40 | $ bazel run :benchmark -- --
41 |
42 | ```
43 |
44 | For example, to benchmark the performance of 2 bazel commits A and B on the same
45 | command `bazel build --nobuild //:all` of `rules_cc` project, you'd do:
46 |
47 | ```shell
48 | $ bazel run :benchmark \
49 | -- \
50 | --bazel_commits=A,B \
51 | --project_source=https://github.com/bazelbuild/rules_cc.git \
52 | -- build --nobuild //:all
53 | ```
54 |
55 | Note the double-dash `--` before the command arguments. You can pass any
56 | arguments that you would normally run on Bazel to the script. The performance of
57 | commands other than `build` can also be benchmarked e.g. `query`, ...
58 |
59 | ### Config-file Interface
60 |
61 | The flag-based approach does not support cases where the benchmarked Bazel
62 | commands differ. The most common use case for this: As a rule developer, I want
63 | to verify the effect of my flag on Bazel performance. For that, we'd need the
64 | config-file interface. The example config file would look like this:
65 |
66 | ```yaml
67 | # config.yaml
68 | global_options:
69 | project_commit: 595a730
70 | runs: 5
71 | collect_profile: false
72 | project_source: /path/to/project/repo
73 | units:
74 | - bazel_binary: /usr/bin/bazel
75 | command: --startup_option1 build --nomy_flag //:all
76 | - bazel_binary: /usr/bin/bazel
77 | command: --startup_option2 build --my_flag //:all
78 | ```
79 |
80 | To launch the benchmark:
81 |
82 | ```shell
83 | $ bazel run :benchmark -- --benchmark_config=/absolute/path/to/config.yaml
84 | ```
85 |
86 | The above config file would benchmark 2 "units". A unit is defined as a set of
87 | conditions that describes a scenario to be benchmarked. This setup allows
88 | maximum flexibility, as the conditions are independent between units. It's even
89 | possible to benchmark a `bazel_commit` against a pre-built `bazel_binary`.
90 |
91 | `global_options` is the list of options applied to every units. These global options are overridden by local options.
92 |
93 | For the list of currently supported flags/attributes and their default values,
94 | refer to [utils/benchmark_config.py](utils/benchmark_config.py).
95 |
96 | #### Known Limitations:
97 |
98 | - `project_source` should be a global option, as we don't support benchmarking
99 | multiple projects in 1 benchmark. Though, `project_commit` can differ between units.
100 | - Incremental benchmarks isn't available.
101 | - Commands have to be in canonical form (next section).
102 |
103 |
104 | ### Bazel Arguments Interpretation
105 |
106 | Bazel arguments are parsed manually. It
107 | is _important_ that the supplied arguments in the command line strictly follows
108 | the canonical form:
109 |
110 | ```
111 |
112 | ```
113 |
114 | Example of non-canonical command line arguments that could result in wrong
115 | interpretation:
116 |
117 | ```
118 | GOOD: (correct order, options in canonical form)
119 | build --nobuild --compilation_mode=opt //:all
120 |
121 | BAD: (non-canonical options)
122 | build --nobuild -c opt //:all
123 |
124 | BAD: (wrong order)
125 | build --nobuild //:all --compilation_mode=opt
126 | ```
127 |
128 | ## Available flags
129 |
130 | To show all the available flags:
131 |
132 | ```
133 | $ bazel run :benchmark -- --helpshort
134 | ```
135 |
136 | Some useful flags are:
137 |
138 | ```
139 | --bazel_binaries: The pre-built bazel binaries to benchmark.
140 | (a comma separated list)
141 | --bazel_commits: The commits at which bazel is built.
142 | (default: 'latest')
143 | (a comma separated list)
144 | --bazel_source: Either a path to the local Bazel repo or a https url to a GitHub repository.
145 | (default: 'https://github.com/bazelbuild/bazel.git')
146 | --bazelrc: The path to a .bazelrc file.
147 | --csv_file_name: The name of the output csv, without the .csv extension
148 | --data_directory: The directory in which the csv files should be stored.
149 | --[no]prefetch_ext_deps: Whether to do an initial run to pre-fetch external dependencies.
150 | (default: 'true')
151 | --project_commits: The commits from the git project to be benchmarked.
152 | (default: 'latest')
153 | (a comma separated list)
154 | --project_source: Either a path to the local git project to be built or a https url to a GitHub repository.
155 | --runs: The number of benchmark runs.
156 | (default: '5')
157 | (an integer)
158 | --[no]verbose: Whether to include git/Bazel stdout logs.
159 | (default: 'false')
160 | --[no]collect_profile: Whether to collect JSON profile for each run.
161 | Requires --data_directory to be set.
162 | (default: 'false')
163 | ```
164 |
165 | ## Collecting JSON Profile
166 |
167 | [Bazel's JSON Profile](https://docs.bazel.build/versions/master/skylark/performance.html#json-profile)
168 | is a useful tool to investigate the performance of Bazel. You can configure
169 | `bazel-bench` to export these JSON profiles on runs using the
170 | `--collect_profile` flag.
171 |
172 | ### JSON Profile Aggregation
173 |
174 | For each pair of `project_commit` and `bazel_commit`, we produce a couple JSON
175 | profiles, based on the number of runs. To have a better overview of the
176 | performance of each phase and events, we can aggregate these profiles and
177 | produce the median duration of each event across them.
178 |
179 | To run the tool:
180 |
181 | ```
182 | bazel run utils:json_profiles_merger \
183 | -- \
184 | --bazel_source= \
185 | --project_source= \
186 | --project_commit= \
187 | --output_path=/tmp/outfile.csv \
188 | -- /tmp/my_json_profiles_*.profile
189 | ```
190 |
191 | You can pass the pattern that selects the input profiles into the positional
192 | argument of the script, like in the above example
193 | (`/tmp/my_json_profiles_*.profile`).
194 |
195 | ## Output Directory Layout
196 |
197 | By default, bazel-bench will store the measurement results and other required
198 | files (project clones, built binaries, ...) under the `~/.bazel-bench`
199 | directory.
200 |
201 | The layout is:
202 |
203 | ```
204 | ~/.bazel-bench/ <= The root of bazel-bench's output dir.
205 | bazel/ <= Where bazel's repository is cloned.
206 | bazel-bin/ <= Where the built bazel binaries are stored.
207 | fba9a2c87ee9589d72889caf082f1029/ <= The bazel commit hash.
208 | bazel <= The actual bazel binary.
209 | project-clones/ <= Where the projects' repositories are cloned.
210 | 7ffd56a6e4cb724ea575aba15733d113/ <= Each project is stored under a project hash,
211 | computed from its source.
212 | out/ <= This is the default output root. But
213 | the output root can also be set via --data_directory.
214 | ```
215 |
216 | To clear the caches, simply `rm -rf` where necessary.
217 |
218 | ## Uploading to BigQuery & Storage
219 |
220 | As an important part of our bazel-bench daily pipeline, we upload the csv output
221 | files to BigQuery and Storage, using separate targets.
222 |
223 | To upload the output to BigQuery & Storage you'll need the GCP credentials and
224 | the table details. Please contact leba@google.com.
225 |
226 | BigQuery:
227 |
228 | ```
229 | bazel run utils:bigquery_upload \
230 | -- \
231 | --upload_to_bigquery=::: \
232 | -- \
233 | ...
234 | ```
235 |
236 | Storage:
237 |
238 | ```
239 | bazel run utils:storage_upload \
240 | -- \
241 | --upload_to_storage=:: \
242 | -- \
243 | ...
244 | ```
245 |
246 | ## Performance Report
247 |
248 | We generate a performance report with BazelCI. The generator script can be found
249 | under the `/report` directory.
250 |
251 | Example Usage: `$ python3 report/generate_report.py --date=2019-01-01
252 | --project=dummy --storage_bucket=dummy_bucket`
253 |
254 | For more detailed usage information, run: `$ python3 report/generate_report.py
255 | --help`
256 |
257 | ## Tests
258 |
259 | The tests for each module are found in the same directory. To run the test,
260 | simply:
261 |
262 | ```
263 | $ bazel test ...
264 | ```
265 |
--------------------------------------------------------------------------------
/WORKSPACE:
--------------------------------------------------------------------------------
1 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
2 |
3 | http_archive(
4 | name = "rules_python",
5 | sha256 = "e85ae30de33625a63eca7fc40a94fea845e641888e52f32b6beea91e8b1b2793",
6 | strip_prefix = "rules_python-0.27.1",
7 | url = "https://github.com/bazelbuild/rules_python/releases/download/0.27.1/rules_python-0.27.1.tar.gz",
8 | )
9 |
10 | load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
11 |
12 | py_repositories()
13 |
14 | load("@rules_python//python:pip.bzl", "pip_parse")
15 |
16 | # Use a hermetic Python interpreter so that builds are reproducible
17 | # irrespective of the Python version available on the host machine.
18 | python_register_toolchains(
19 | name = "python3_10",
20 | python_version = "3.10",
21 | )
22 |
23 | load("@python3_10//:defs.bzl", "interpreter")
24 |
25 | # Translate requirements.txt into a @third_party external repository.
26 | pip_parse(
27 | name = "third_party",
28 | python_interpreter_target = interpreter,
29 | requirements_lock = "//third_party:requirements.txt",
30 | )
31 |
32 | load("@third_party//:requirements.bzl", "install_deps")
33 |
34 | #
35 | install_deps()
36 |
--------------------------------------------------------------------------------
/WORKSPACE.bzlmod:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bazelbuild/bazel-bench/f0c8f585ad4733f184222be59c4401f9371991a6/WORKSPACE.bzlmod
--------------------------------------------------------------------------------
/bb-icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bazelbuild/bazel-bench/f0c8f585ad4733f184222be59c4401f9371991a6/bb-icon.png
--------------------------------------------------------------------------------
/benchmark.py:
--------------------------------------------------------------------------------
1 | # Copyright 2019 The Bazel Authors. All rights reserved.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http:#www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | import csv
15 | import collections
16 | import datetime
17 | import os
18 | import subprocess
19 | import sys
20 | import hashlib
21 | import re
22 | import shutil
23 | import collections
24 | import tempfile
25 | import git
26 | import utils.logger as logger
27 | import utils.json_profiles_merger_lib as json_profiles_merger_lib
28 | import utils.output_handling as output_handling
29 |
30 | from absl import app
31 | from absl import flags
32 |
33 | from utils.values import Values
34 | from utils.bazel import Bazel
35 | from utils.benchmark_config import BenchmarkConfig
36 |
37 | # BB_ROOT has different values, depending on the platform.
38 | BB_ROOT = os.path.join(os.path.expanduser('~'), '.bazel-bench')
39 |
40 | # The path to the directory that stores Bazel clones.
41 | BAZEL_CLONE_BASE_PATH = os.path.join(BB_ROOT, 'bazel-clones')
42 | # The path to the directory that stores project clones.
43 | PROJECT_CLONE_BASE_PATH = os.path.join(BB_ROOT, 'project-clones')
44 | BAZEL_GITHUB_URL = 'https://github.com/bazelbuild/bazel.git'
45 | # The path to the directory that stores the bazel binaries.
46 | BAZEL_BINARY_BASE_PATH = os.path.join(BB_ROOT, 'bazel-bin')
47 | # The path to the directory that stores the output csv (If required).
48 | DEFAULT_OUT_BASE_PATH = os.path.join(BB_ROOT, 'out')
49 | # The default name of the aggr json profile.
50 | DEFAULT_AGGR_JSON_PROFILE_FILENAME = 'aggr_json_profiles.csv'
51 |
52 |
53 | def _get_clone_subdir(project_source):
54 | """Calculates a hexdigest of project_source to serve as a unique subdir name."""
55 | return hashlib.md5(project_source.encode('utf-8')).hexdigest()
56 |
57 |
58 | def _exec_command(args, shell=False, cwd=None):
59 | logger.log('Executing: %s' % (args if shell else ' '.join(args)))
60 |
61 | return subprocess.run(
62 | args,
63 | shell=shell,
64 | cwd=cwd,
65 | check=True,
66 | stdout=sys.stdout if FLAGS.verbose else subprocess.DEVNULL,
67 | stderr=sys.stderr if FLAGS.verbose else subprocess.DEVNULL)
68 |
69 |
70 | def _get_commits_topological(commits_sha_list,
71 | repo,
72 | flag_name,
73 | fill_default=True):
74 | """Returns a list of commits, sorted by topological order.
75 |
76 | e.g. for a commit history A -> B -> C -> D, commits_sha_list = [C, B]
77 | Output: [B, C]
78 |
79 | If the input commits_sha_list is empty, fetch the latest commit on branch
80 | 'master'
81 | of the repo.
82 |
83 | Args:
84 | commits_sha_list: a list of string of commit SHA digest. Can be long or
85 | short digest.
86 | repo: the git.Repo instance of the repository.
87 | flag_name: the flag that is supposed to specify commits_list.
88 | fill_default: whether to fill in a default latest commit if none is
89 | specified.
90 |
91 | Returns:
92 | A list of string of full SHA digests, sorted by topological commit order.
93 | """
94 | if commits_sha_list:
95 | long_commits_sha_set = set(
96 | map(lambda x: _to_long_sha_digest(x, repo), commits_sha_list))
97 | sorted_commit_list = []
98 | for c in reversed(list(repo.iter_commits())):
99 | if c.hexsha in long_commits_sha_set:
100 | sorted_commit_list.append(c.hexsha)
101 |
102 | if len(sorted_commit_list) != len(long_commits_sha_set):
103 | raise ValueError(
104 | "The following commits weren't found in the repo in branch master: %s."
105 | % (long_commits_sha_set - set(sorted_commit_list)))
106 | return sorted_commit_list
107 |
108 | elif not fill_default:
109 | # If we have some binary paths specified, we don't need to fill in a default
110 | # commit.
111 | return []
112 |
113 | # If no commit specified: take the repo's latest commit.
114 | latest_commit_sha = repo.commit().hexsha
115 | logger.log('No %s specified, using the latest one: %s' %
116 | (flag_name, latest_commit_sha))
117 | return [latest_commit_sha]
118 |
119 |
120 | def _to_long_sha_digest(digest, repo):
121 | """Returns the full 40-char SHA digest of a commit."""
122 | return repo.git.rev_parse(digest) if len(digest) < 40 else digest
123 |
124 |
125 | def _setup_project_repo(repo_path, project_source):
126 | """Returns a path to the cloned repository.
127 |
128 | If the repo_path exists, perform a `git fetch` to update the content.
129 | Else, clone the project to repo_path.
130 |
131 | Args:
132 | repo_path: the path to clone the repository to.
133 | project_source: the source to clone the repository from. Could be a local
134 | path or an URL.
135 |
136 | Returns:
137 | A git.Repo object of the cloned repository.
138 | """
139 | if os.path.exists(repo_path):
140 | logger.log('Path %s exists. Updating...' % repo_path)
141 | repo = git.Repo(repo_path)
142 | repo.git.fetch('origin')
143 | else:
144 | logger.log('Cloning %s to %s...' % (project_source, repo_path))
145 | repo = git.Repo.clone_from(project_source, repo_path)
146 |
147 | return repo
148 |
149 |
150 | def _build_bazel_binary(commit, repo, outroot, platform=None):
151 | """Builds bazel at the specified commit and copy the output binary to outroot.
152 |
153 | If the binary for this commit already exists at the destination path, simply
154 | return the path without re-building.
155 |
156 | Args:
157 | commit: the Bazel commit SHA.
158 | repo: the git.Repo instance of the Bazel clone.
159 | outroot: the directory inwhich the resulting binary is copied to.
160 | platform: the platform on which to build this binary.
161 |
162 | Returns:
163 | The path to the resulting binary (copied to outroot).
164 | """
165 | outroot_for_commit = '%s/%s/%s' % (
166 | outroot, platform, commit) if platform else '%s/%s' % (outroot, commit)
167 | destination = '%s/bazel' % outroot_for_commit
168 | if os.path.exists(destination):
169 | logger.log('Binary exists at %s, reusing...' % destination)
170 | return destination
171 |
172 | logger.log('Building Bazel binary at commit %s' % commit)
173 | repo.git.checkout('-f', commit)
174 |
175 | _exec_command(['bazel', 'build', '//src:bazel'], cwd=repo.working_dir)
176 |
177 | # Copy to another location
178 | binary_out = '%s/bazel-bin/src/bazel' % repo.working_dir
179 |
180 | if not os.path.exists(outroot_for_commit):
181 | os.makedirs(outroot_for_commit)
182 | logger.log('Copying bazel binary to %s' % destination)
183 | shutil.copyfile(binary_out, destination)
184 | _exec_command(['chmod', '+x', destination])
185 |
186 | return destination
187 |
188 |
189 | def _construct_json_profile_flags(out_file_path):
190 | """Constructs the flags used to collect JSON profiles.
191 |
192 | Args:
193 | out_file_path: The path to output the profile to.
194 |
195 | Returns:
196 | A list of string representing the flags.
197 | """
198 | return [
199 | '--experimental_generate_json_trace_profile',
200 | '--profile={}'.format(out_file_path)
201 | ]
202 |
203 |
204 | def json_profile_filename(data_directory, bazel_bench_uid, bazel_commit,
205 | unit_num, project_commit, run_number, total_runs):
206 | return (f'{data_directory}/{bazel_bench_uid}_{bazel_commit}_{unit_num}'
207 | + f'_{project_commit}_{run_number}_of_{total_runs}.profile.gz')
208 |
209 |
210 | def _single_run(bazel_bin_path,
211 | command,
212 | options,
213 | targets,
214 | startup_options):
215 | """Runs the benchmarking for a combination of (bazel version, project version).
216 |
217 | Args:
218 | bazel_bin_path: the path to the bazel binary to be run.
219 | command: the command to be run with Bazel.
220 | options: the list of options.
221 | targets: the list of targets.
222 | startup_options: the list of target options.
223 |
224 | Returns:
225 | A result object:
226 | {
227 | 'wall': 1.000,
228 | 'cpu': 1.000,
229 | 'system': 1.000,
230 | 'memory': 1.000,
231 | 'exit_status': 0,
232 | 'started_at': datetime.datetime(2019, 1, 1, 0, 0, 0, 000000),
233 | }
234 | """
235 | bazel = Bazel(bazel_bin_path, startup_options)
236 |
237 | default_arguments = collections.defaultdict(list)
238 |
239 | # Prepend some default options if the command is 'build'.
240 | # The order in which the options appear matters.
241 | if command == 'build':
242 | options = options + ['--nostamp', '--noshow_progress', '--color=no']
243 | measurements = bazel.command(command, args=options + targets)
244 |
245 | if measurements != None:
246 | logger.log('Results of this run: wall: ' +
247 | '%.3fs, cpu %.3fs, system %.3fs, memory %.3fMB, exit_status: %d' % (
248 | measurements['wall'],
249 | measurements['cpu'],
250 | measurements['system'],
251 | measurements['memory'],
252 | measurements['exit_status']))
253 |
254 | if FLAGS.clean:
255 | bazel.command('clean', ['--color=no'])
256 |
257 | if FLAGS.shutdown:
258 | bazel.command('shutdown')
259 |
260 | return measurements
261 |
262 |
263 | def _run_benchmark(bazel_bin_path,
264 | project_path,
265 | runs,
266 | command,
267 | options,
268 | targets,
269 | startup_options,
270 | prefetch_ext_deps,
271 | bazel_bench_uid,
272 | unit_num,
273 | data_directory=None,
274 | collect_profile=False,
275 | bazel_identifier=None,
276 | project_commit=None):
277 | """Runs the benchmarking for a combination of (bazel version, project version).
278 |
279 | Args:
280 | bazel_bin_path: the path to the bazel binary to be run.
281 | project_path: the path to the project clone to be built.
282 | runs: the number of runs.
283 | bazel_args: the unparsed list of arguments to be passed to Bazel binary.
284 | prefetch_ext_deps: whether to do a first non-benchmarked run to fetch the
285 | external dependencies.
286 | bazel_bench_uid: a unique string identifier of this entire bazel-bench run.
287 | unit_num: the numerical order of the current unit being benchmarked.
288 | collect_profile: whether to collect JSON profile for each run.
289 | data_directory: the path to the directory to store run data. Required if
290 | collect_profile.
291 | bazel_identifier: the commit hash of the bazel commit. Required if
292 | collect_profile.
293 | project_commit: the commit hash of the project commit. Required if
294 | collect_profile.
295 |
296 | Returns:
297 | A list of result objects from each _single_run.
298 | """
299 | collected = []
300 | os.chdir(project_path)
301 |
302 | logger.log('=== BENCHMARKING BAZEL [Unit #%d]: %s, PROJECT: %s ===' %
303 | (unit_num, bazel_identifier, project_commit))
304 | # Runs the command once to make sure external dependencies are fetched.
305 | if prefetch_ext_deps:
306 | logger.log('Pre-fetching external dependencies...')
307 | _single_run(bazel_bin_path, command, options, targets, startup_options)
308 |
309 | if collect_profile:
310 | if not os.path.exists(data_directory):
311 | os.makedirs(data_directory)
312 |
313 | for i in range(1, runs + 1):
314 | logger.log('Starting benchmark run %s/%s:' % (i, runs))
315 |
316 | maybe_include_json_profile_flags = options[:]
317 | if collect_profile:
318 | assert bazel_identifier, ('bazel_identifier is required when '
319 | 'collect_profile')
320 | assert project_commit, ('project_commit is required when '
321 | 'collect_profile')
322 | maybe_include_json_profile_flags += _construct_json_profile_flags(
323 | json_profile_filename(
324 | data_directory=data_directory,
325 | bazel_bench_uid=bazel_bench_uid,
326 | bazel_commit=bazel_identifier.replace('/', '_'),
327 | unit_num=unit_num,
328 | project_commit=project_commit,
329 | run_number=i,
330 | total_runs=runs,
331 | ))
332 | collected.append(
333 | _single_run(bazel_bin_path, command, maybe_include_json_profile_flags,
334 | targets, startup_options))
335 |
336 | return collected, (command, targets, options)
337 |
338 |
339 | def handle_json_profiles_aggr(bazel_bench_uid, unit_num, bazel_commits,
340 | project_source, project_commits, runs,
341 | output_path, data_directory):
342 | """Aggregates the collected JSON profiles and writes the result to a CSV.
343 |
344 | Args:
345 | bazel_bench_uid: a unique string identifier of this entire bazel-bench run.
346 | unit_num: the numerical order of the current unit being benchmarked.
347 | bazel_commits: the Bazel commits that bazel-bench ran on.
348 | project_source: a path/url to a local/remote repository of the project on
349 | which benchmarking was performed.
350 | project_commits: the commits of the project when benchmarking was done.
351 | runs: the total number of runs.
352 | output_path: the path to the output csv file.
353 | data_directory: the directory that stores output files.
354 | """
355 | output_dir = os.path.dirname(output_path)
356 | if not os.path.exists(output_dir):
357 | os.makedirs(output_dir)
358 |
359 | with open(output_path, 'w') as f:
360 | csv_writer = csv.writer(f)
361 | csv_writer.writerow([
362 | 'bazel_source', 'project_source', 'project_commit', 'cat', 'name', 'dur'
363 | ])
364 |
365 | for bazel_commit in bazel_commits:
366 | for project_commit in project_commits:
367 | profiles_filenames = [
368 | json_profile_filename(
369 | data_directory=data_directory,
370 | bazel_bench_uid=bazel_bench_uid,
371 | bazel_commit=bazel_commit,
372 | unit_num=unit_num,
373 | project_commit=project_commit,
374 | run_number=i,
375 | total_runs=runs,
376 | )
377 | for i in range(1, runs + 1)
378 | ]
379 | event_list = json_profiles_merger_lib.aggregate_data(
380 | profiles_filenames, only_phases=True)
381 | for event in event_list:
382 | csv_writer.writerow([
383 | bazel_commit, project_source, project_commit, event['cat'],
384 | event['name'], event['median']
385 | ])
386 | logger.log('Finished writing aggregate_json_profiles to %s' % output_path)
387 |
388 |
389 | def create_summary(data, project_source):
390 | """Creates the runs summary onto stdout.
391 |
392 | Excludes runs with non-zero exit codes from the final summary table.
393 | """
394 | unit = {
395 | 'wall': 's ',
396 | 'cpu': 's ',
397 | 'system': 's ',
398 | 'memory': 'MB'
399 | }
400 | summary_builder = []
401 | summary_builder.append('\nRESULTS:')
402 | last_collected = None
403 | for (i, bazel_commit, project_commit), collected in data.items():
404 | header = ('[Unit #%d] Bazel version: %s, Project commit: %s, Project source: %s' %
405 | (i, bazel_commit, project_commit, project_source))
406 | summary_builder.append(header)
407 |
408 | summary_builder.append(
409 | '%s %s %s %s %s' %
410 | ('metric'.rjust(8), 'mean'.center(20), 'median'.center(20),
411 | 'stddev'.center(10), 'pval'.center(10)))
412 |
413 | num_runs = len(collected['wall'].items())
414 | # A map from run number to exit code, for runs with non-zero exit codes.
415 | non_zero_runs = {}
416 | for i, exit_code in enumerate(collected['exit_status'].items()):
417 | if exit_code != 0:
418 | non_zero_runs[i] = exit_code
419 | for metric, values in collected.items():
420 | if metric in ['exit_status', 'started_at']:
421 | continue
422 |
423 | values_exclude_failures = values.exclude_from_indexes(
424 | non_zero_runs.keys())
425 | # Skip if there's no value available after excluding failed runs.
426 | if not values_exclude_failures.items():
427 | continue
428 |
429 | if last_collected:
430 | base = last_collected[metric]
431 | pval = '% 7.5f' % values_exclude_failures.pval(base.values())
432 | mean_diff = '(% +6.2f%%)' % (
433 | 100. * (values_exclude_failures.mean() - base.mean()) / base.mean())
434 | median_diff = '(% +6.2f%%)' % (
435 | 100. *
436 | (values_exclude_failures.median() - base.median()) / base.median())
437 | else:
438 | pval = ''
439 | mean_diff = median_diff = ' '
440 | summary_builder.append(
441 | '%s: %s %s %s %s' %
442 | (metric.rjust(8),
443 | ('% 8.3f%s %s' %
444 | (values_exclude_failures.mean(), unit[metric], mean_diff)).center(20),
445 | ('% 8.3f%s %s' %
446 | (values_exclude_failures.median(), unit[metric], median_diff)).center(20),
447 | ('% 7.3f%s' % (values_exclude_failures.stddev(), unit[metric])).center(10),
448 | pval.center(10)))
449 | last_collected = collected
450 | if non_zero_runs:
451 | summary_builder.append(
452 | ('The following runs contain non-zero exit code(s):\n %s\n'
453 | 'Please check the full log for more details. These runs are '
454 | 'excluded from the above result table.' %
455 | '\n '.join('- run: %s/%s, exit_code: %s' % (k + 1, num_runs, v)
456 | for k, v in non_zero_runs.items())))
457 | summary_builder.append('')
458 |
459 | return '\n'.join(summary_builder)
460 |
461 |
462 | FLAGS = flags.FLAGS
463 | # Flags for the bazel binaries.
464 | flags.DEFINE_list('bazel_commits', None, 'The commits at which bazel is built.')
465 | flags.DEFINE_list('bazel_binaries', None,
466 | 'The pre-built bazel binaries to benchmark.')
467 | flags.DEFINE_string('bazel_source',
468 | 'https://github.com/bazelbuild/bazel.git',
469 | 'Either a path to the local Bazel repo or a https url to ' \
470 | 'a GitHub repository.')
471 | flags.DEFINE_string(
472 | 'bazel_bin_dir', None,
473 | 'The directory to store the bazel binaries from each commit.')
474 |
475 | # Flags for the project to be built.
476 | flags.DEFINE_string(
477 | 'project_label', None,
478 | 'The label of the project. Only relevant in the daily performance report.')
479 | flags.DEFINE_string('project_source', None,
480 | 'Either a path to the local git project to be built or ' \
481 | 'a https url to a GitHub repository.')
482 | flags.DEFINE_list('project_commits', None,
483 | 'The commits from the git project to be benchmarked.')
484 | flags.DEFINE_string(
485 | 'env_configure', None,
486 | "The shell commands to configure the project's environment.")
487 |
488 | # Execution options.
489 | flags.DEFINE_integer('runs', 5, 'The number of benchmark runs.')
490 | flags.DEFINE_string('bazelrc', None, 'The path to a .bazelrc file.')
491 | flags.DEFINE_string('platform', None,
492 | ('The platform on which bazel-bench is run. This is just '
493 | 'to categorize data and has no impact on the actual '
494 | 'script execution.'))
495 | flags.DEFINE_boolean('clean', True, 'Whether to invoke clean between runs/builds.')
496 | flags.DEFINE_boolean('shutdown', True, 'Whether to invoke shutdown between runs/builds.')
497 |
498 | # Miscellaneous flags.
499 | flags.DEFINE_boolean('verbose', False,
500 | 'Whether to include git/Bazel stdout logs.')
501 | flags.DEFINE_boolean('prefetch_ext_deps', True,
502 | 'Whether to do an initial run to pre-fetch external ' \
503 | 'dependencies.')
504 | flags.DEFINE_boolean('collect_profile', False,
505 | 'Whether to collect JSON profile for each run. Requires ' \
506 | '--data_directory to be set.')
507 | flags.DEFINE_boolean('aggregate_json_profiles', False,
508 | 'Whether to aggregate the collected JSON profiles. Requires '\
509 | '--collect_profile to be set.')
510 | flags.DEFINE_string(
511 | 'benchmark_config', None,
512 | 'Whether to use the config-file interface to define benchmark units.')
513 |
514 | # Output storage flags.
515 | flags.DEFINE_string('data_directory', None,
516 | 'The directory in which the csv files should be stored.')
517 | # The daily report generation process on BazelCI requires the csv file name to
518 | # be determined before bazel-bench is launched, so that METADATA files are
519 | # properly filled.
520 | flags.DEFINE_string('csv_file_name', None,
521 | 'The name of the output csv, without the .csv extension.')
522 |
523 |
524 | def _flag_checks():
525 | """Verify flags requirements."""
526 | if (not FLAGS.benchmark_config and FLAGS.bazel_commits and
527 | FLAGS.project_commits and len(FLAGS.bazel_commits) > 1 and
528 | len(FLAGS.project_commits) > 1):
529 | raise ValueError(
530 | 'Either --bazel_commits or --project_commits should be a single element.'
531 | )
532 |
533 | if FLAGS.aggregate_json_profiles and not FLAGS.collect_profile:
534 | raise ValueError('--aggregate_json_profiles requires '
535 | '--collect_profile to be set.')
536 |
537 |
538 | def _get_benchmark_config_and_clone_repos(argv):
539 | """From the flags/config file, get the benchmark units.
540 |
541 | Args:
542 | argv: the command line arguments.
543 |
544 | Returns:
545 | An instance of BenchmarkConfig that contains the benchmark units.
546 | """
547 | if FLAGS.benchmark_config:
548 | config = BenchmarkConfig.from_file(FLAGS.benchmark_config)
549 | project_source = config.get_project_source()
550 | project_clone_repo = _setup_project_repo(
551 | PROJECT_CLONE_BASE_PATH + '/' + _get_clone_subdir(project_source),
552 | project_source)
553 | bazel_source = config.get_bazel_source()
554 | bazel_clone_repo = _setup_project_repo(
555 | BAZEL_CLONE_BASE_PATH + '/' + _get_clone_subdir(bazel_source),
556 | bazel_source)
557 |
558 | return config, bazel_clone_repo, project_clone_repo
559 |
560 | # Strip off 'benchmark.py' from argv
561 | # argv would be something like:
562 | # ['benchmark.py', 'build', '--nobuild', '//:all']
563 | bazel_args = argv[1:]
564 |
565 | # Building Bazel binaries
566 | bazel_binaries = FLAGS.bazel_binaries or []
567 | logger.log('Preparing bazelbuild/bazel repository.')
568 | bazel_source = FLAGS.bazel_source if FLAGS.bazel_source else BAZEL_GITHUB_URL
569 | bazel_clone_repo = _setup_project_repo(
570 | PROJECT_CLONE_BASE_PATH + '/' + _get_clone_subdir(bazel_source),
571 | bazel_source)
572 | bazel_commits = _get_commits_topological(
573 | FLAGS.bazel_commits,
574 | bazel_clone_repo,
575 | 'bazel_commits',
576 | fill_default=not FLAGS.bazel_commits and not bazel_binaries)
577 |
578 | # Set up project repo
579 | logger.log('Preparing %s clone.' % FLAGS.project_source)
580 | project_clone_repo = _setup_project_repo(
581 | PROJECT_CLONE_BASE_PATH + '/' + _get_clone_subdir(FLAGS.project_source),
582 | FLAGS.project_source)
583 |
584 | project_commits = _get_commits_topological(FLAGS.project_commits,
585 | project_clone_repo,
586 | 'project_commits')
587 |
588 | config = BenchmarkConfig.from_flags(
589 | bazel_commits=bazel_commits,
590 | bazel_binaries=bazel_binaries,
591 | project_commits=project_commits,
592 | bazel_source=bazel_source,
593 | project_source=FLAGS.project_source,
594 | env_configure=FLAGS.env_configure,
595 | runs=FLAGS.runs,
596 | collect_profile=FLAGS.collect_profile,
597 | command=' '.join(bazel_args),
598 | clean=FLAGS.clean,
599 | shutdown=FLAGS.shutdown)
600 |
601 | return config, bazel_clone_repo, project_clone_repo
602 |
603 |
604 | def main(argv):
605 | _flag_checks()
606 |
607 | config, bazel_clone_repo, project_clone_repo = _get_benchmark_config_and_clone_repos(
608 | argv)
609 |
610 | # A dictionary that maps a (bazel_commit, project_commit) tuple
611 | # to its benchmarking result.
612 | data = collections.OrderedDict()
613 | csv_data = collections.OrderedDict()
614 | data_directory = FLAGS.data_directory or DEFAULT_OUT_BASE_PATH
615 |
616 | # We use the start time as a unique identifier of this bazel-bench run.
617 | bazel_bench_uid = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')
618 |
619 | bazel_bin_base_path = FLAGS.bazel_bin_dir or BAZEL_BINARY_BASE_PATH
620 |
621 | # Build the bazel binaries, if necessary.
622 | for unit in config.get_units():
623 | if 'bazel_binary' in unit:
624 | unit['bazel_bin_path'] = unit['bazel_binary']
625 | elif 'bazel_commit' in unit:
626 | bazel_bin_path = _build_bazel_binary(unit['bazel_commit'],
627 | bazel_clone_repo,
628 | bazel_bin_base_path, FLAGS.platform)
629 | unit['bazel_bin_path'] = bazel_bin_path
630 |
631 | for i, unit in enumerate(config.get_units()):
632 | bazel_identifier = unit['bazel_commit'] if 'bazel_commit' in unit else unit['bazel_binary']
633 | project_commit = unit['project_commit']
634 |
635 | project_clone_repo.git.checkout('-f', project_commit)
636 | if unit['env_configure'] is not None:
637 | _exec_command(
638 | unit['env_configure'], shell=True, cwd=project_clone_repo.working_dir)
639 |
640 | results, args = _run_benchmark(
641 | bazel_bin_path=unit['bazel_bin_path'],
642 | project_path=project_clone_repo.working_dir,
643 | runs=unit['runs'],
644 | command=unit['command'],
645 | options=unit['options'],
646 | targets=unit['targets'],
647 | startup_options=unit['startup_options'],
648 | prefetch_ext_deps=FLAGS.prefetch_ext_deps,
649 | bazel_bench_uid=bazel_bench_uid,
650 | unit_num=i,
651 | collect_profile=unit['collect_profile'],
652 | data_directory=data_directory,
653 | bazel_identifier=bazel_identifier,
654 | project_commit=project_commit)
655 | collected = {}
656 | for benchmarking_result in results:
657 | for metric, value in benchmarking_result.items():
658 | if metric not in collected:
659 | collected[metric] = Values()
660 | collected[metric].add(value)
661 |
662 | data[(i, bazel_identifier, project_commit)] = collected
663 | non_measurables = {
664 | 'project_source': unit['project_source'],
665 | 'platform': FLAGS.platform,
666 | 'project_label': FLAGS.project_label
667 | }
668 | csv_data[(bazel_identifier, project_commit)] = {
669 | 'results': results,
670 | 'args': args,
671 | 'non_measurables': non_measurables
672 | }
673 |
674 | summary_text = create_summary(data, config.get_project_source())
675 | print(summary_text)
676 |
677 | if FLAGS.data_directory:
678 | csv_file_name = FLAGS.csv_file_name or '{}.csv'.format(bazel_bench_uid)
679 | txt_file_name = csv_file_name.replace('.csv', '.txt')
680 |
681 | output_handling.export_csv(data_directory, csv_file_name, csv_data)
682 | output_handling.export_file(data_directory, txt_file_name, summary_text)
683 |
684 | # This is mostly for the nightly benchmark.
685 | if FLAGS.aggregate_json_profiles:
686 | aggr_json_profiles_csv_path = (
687 | '%s/%s' % (FLAGS.data_directory, DEFAULT_AGGR_JSON_PROFILE_FILENAME))
688 | handle_json_profiles_aggr(
689 | bazel_bench_uid=bazel_bench_uid,
690 | unit_num=i,
691 | bazel_commits=config.get_bazel_commits(),
692 | project_source=config.get_project_source(),
693 | project_commits=config.get_project_commits(),
694 | runs=FLAGS.runs,
695 | output_path=aggr_json_profiles_csv_path,
696 | data_directory=FLAGS.data_directory,
697 | )
698 |
699 | logger.log('Done.')
700 |
701 |
702 | if __name__ == '__main__':
703 | app.run(main)
704 |
--------------------------------------------------------------------------------
/benchmark_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2019 The Bazel Authors. All rights reserved.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http:#www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | """Tests for the main benchmarking script."""
15 | import mock
16 | import sys
17 | import benchmark
18 | import six
19 |
20 | from absl.testing import absltest
21 | from absl.testing import flagsaver
22 | from absl import flags
23 | from testutils.fakes import fake_log, fake_exec_command, FakeBazel
24 |
25 | # Setup custom fakes/mocks.
26 | benchmark.logger.log = fake_log
27 | benchmark._exec_command = fake_exec_command
28 | benchmark.Bazel = FakeBazel
29 | mock_stdio_type = six.StringIO
30 |
31 |
32 | class BenchmarkFunctionTests(absltest.TestCase):
33 |
34 | @mock.patch.object(benchmark.os.path, 'exists', return_value=True)
35 | @mock.patch.object(benchmark.os, 'chdir')
36 | def test_setup_project_repo_exists(self, unused_chdir_mock,
37 | unused_exists_mock):
38 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr, \
39 | mock.patch('benchmark.git.Repo') as mock_repo_class:
40 | mock_repo = mock_repo_class.return_value
41 | benchmark._setup_project_repo('repo_path', 'project_source')
42 |
43 | mock_repo.git.fetch.assert_called_once_with('origin')
44 | self.assertEqual('Path repo_path exists. Updating...',
45 | mock_stderr.getvalue())
46 |
47 | @mock.patch.object(benchmark.os.path, 'exists', return_value=False)
48 | @mock.patch.object(benchmark.os, 'chdir')
49 | def test_setup_project_repo_not_exists(self, unused_chdir_mock,
50 | unused_exists_mock):
51 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr, \
52 | mock.patch('benchmark.git.Repo') as mock_repo_class:
53 | benchmark._setup_project_repo('repo_path', 'project_source')
54 |
55 | mock_repo_class.clone_from.assert_called_once_with('project_source',
56 | 'repo_path')
57 | self.assertEqual('Cloning project_source to repo_path...',
58 | mock_stderr.getvalue())
59 |
60 | def test_get_commits_topological(self):
61 | with mock.patch('benchmark.git.Repo') as mock_repo_class:
62 | mock_repo = mock_repo_class.return_value
63 | mock_A = mock.MagicMock()
64 | mock_A.hexsha = 'A'
65 | mock_B = mock.MagicMock()
66 | mock_B.hexsha = 'B'
67 | mock_C = mock.MagicMock()
68 | mock_C.hexsha = 'C'
69 | mock_repo.iter_commits.return_value = [mock_C, mock_B, mock_A]
70 | mock_repo.git.rev_parse.side_effect = lambda x: x
71 | result = benchmark._get_commits_topological(['B', 'A'], mock_repo,
72 | 'flag_name')
73 |
74 | self.assertEqual(['A', 'B'], result)
75 |
76 | def test_get_commits_topological_latest(self):
77 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr, \
78 | mock.patch('benchmark.git.Repo') as mock_repo_class:
79 | mock_repo = mock_repo_class.return_value
80 | mock_commit = mock.MagicMock()
81 | mock_repo.commit.return_value = mock_commit
82 | mock_commit.hexsha = 'A'
83 | result = benchmark._get_commits_topological(None, mock_repo,
84 | 'bazel_commits')
85 |
86 | self.assertEqual(['A'], result)
87 | self.assertEqual('No bazel_commits specified, using the latest one: A',
88 | mock_stderr.getvalue())
89 |
90 | @mock.patch.object(benchmark.os.path, 'exists', return_value=True)
91 | @mock.patch.object(benchmark.os, 'makedirs')
92 | def test_build_bazel_binary_exists(self, unused_chdir_mock,
93 | unused_exists_mock):
94 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr:
95 | benchmark._build_bazel_binary('commit', 'repo_path', 'outroot')
96 | self.assertEqual('Binary exists at outroot/commit/bazel, reusing...',
97 | mock_stderr.getvalue())
98 |
99 | @mock.patch.object(benchmark.os.path, 'exists', return_value=False)
100 | @mock.patch.object(benchmark.os, 'makedirs')
101 | @mock.patch.object(benchmark.os, 'chdir')
102 | @mock.patch.object(benchmark.shutil, 'copyfile')
103 | def test_build_bazel_binary_not_exists(self, unused_shutil_mock,
104 | unused_chdir_mock,
105 | unused_makedirs_mock,
106 | unused_exists_mock):
107 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr, \
108 | mock.patch('benchmark.git.Repo') as mock_repo_class:
109 | mock_repo = mock_repo_class.return_value
110 | benchmark._build_bazel_binary('commit', mock_repo, 'outroot')
111 |
112 | mock_repo.git.checkout.assert_called_once_with('-f', 'commit')
113 | self.assertEqual(
114 | ''.join([
115 | 'Building Bazel binary at commit commit', 'bazel build //src:bazel',
116 | 'Copying bazel binary to outroot/commit/bazel',
117 | 'chmod +x outroot/commit/bazel'
118 | ]), mock_stderr.getvalue())
119 |
120 | def test_single_run(self):
121 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr:
122 | benchmark._single_run(
123 | 'bazel_binary_path',
124 | 'build',
125 | options=[],
126 | targets=['//:all'],
127 | startup_options=[])
128 |
129 | self.assertEqual(
130 | ''.join([
131 | 'Executing Bazel command: bazel build --nostamp --noshow_progress --color=no //:all',
132 | 'Executing Bazel command: bazel clean --color=no',
133 | 'Executing Bazel command: bazel shutdown '
134 | ]), mock_stderr.getvalue())
135 |
136 | @mock.patch.object(benchmark.os, 'chdir')
137 | def test_run_benchmark_no_prefetch(self, _):
138 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr:
139 | benchmark._run_benchmark(
140 | 'bazel_binary_path',
141 | 'project_path',
142 | runs=2,
143 | bazel_bench_uid='fake_uid',
144 | command='build',
145 | options=[],
146 | targets=['//:all'],
147 | startup_options=[],
148 | prefetch_ext_deps=False,
149 | unit_num=0)
150 |
151 | self.assertEqual(
152 | ''.join([
153 | '=== BENCHMARKING BAZEL [Unit #0]: None, PROJECT: None ===',
154 | 'Starting benchmark run 1/2:',
155 | 'Executing Bazel command: bazel build --nostamp --noshow_progress --color=no //:all',
156 | 'Executing Bazel command: bazel clean --color=no',
157 | 'Executing Bazel command: bazel shutdown ',
158 | 'Starting benchmark run 2/2:',
159 | 'Executing Bazel command: bazel build --nostamp --noshow_progress --color=no //:all',
160 | 'Executing Bazel command: bazel clean --color=no',
161 | 'Executing Bazel command: bazel shutdown '
162 | ]), mock_stderr.getvalue())
163 |
164 | @mock.patch.object(benchmark.os, 'chdir')
165 | def test_run_benchmark_prefetch(self, _):
166 | benchmark.DEFAULT_OUT_BASE_PATH = 'some_out_path'
167 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr:
168 | benchmark._run_benchmark(
169 | 'bazel_binary_path',
170 | 'project_path',
171 | runs=2,
172 | bazel_bench_uid='fake_uid',
173 | command='build',
174 | options=[],
175 | targets=['//:all'],
176 | startup_options=[],
177 | prefetch_ext_deps=True,
178 | unit_num=0)
179 |
180 | self.assertEqual(
181 | ''.join([
182 | '=== BENCHMARKING BAZEL [Unit #0]: None, PROJECT: None ===',
183 | 'Pre-fetching external dependencies...',
184 | 'Executing Bazel command: bazel build --nostamp --noshow_progress --color=no //:all',
185 | 'Executing Bazel command: bazel clean --color=no',
186 | 'Executing Bazel command: bazel shutdown ',
187 | 'Starting benchmark run 1/2:',
188 | 'Executing Bazel command: bazel build --nostamp --noshow_progress --color=no //:all',
189 | 'Executing Bazel command: bazel clean --color=no',
190 | 'Executing Bazel command: bazel shutdown ',
191 | 'Starting benchmark run 2/2:',
192 | 'Executing Bazel command: bazel build --nostamp --noshow_progress --color=no //:all',
193 | 'Executing Bazel command: bazel clean --color=no',
194 | 'Executing Bazel command: bazel shutdown '
195 | ]), mock_stderr.getvalue())
196 |
197 | @mock.patch.object(benchmark.os, 'chdir')
198 | def test_run_benchmark_collect_profile(self, _):
199 | benchmark.DEFAULT_OUT_BASE_PATH = 'some_out_path'
200 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr:
201 | benchmark._run_benchmark(
202 | 'bazel_binary_path',
203 | 'project_path',
204 | runs=2,
205 | bazel_bench_uid='fake_uid',
206 | command='build',
207 | options=[],
208 | targets=['//:all'],
209 | startup_options=[],
210 | prefetch_ext_deps=True,
211 | collect_profile=True,
212 | data_directory='fake_dir',
213 | bazel_identifier='fake_bazel_commit',
214 | project_commit='fake_project_commit',
215 | unit_num=0)
216 |
217 | self.assertEqual(
218 | ''.join([
219 | '=== BENCHMARKING BAZEL [Unit #0]: fake_bazel_commit, PROJECT: fake_project_commit ===',
220 | 'Pre-fetching external dependencies...',
221 | 'Executing Bazel command: bazel build --nostamp --noshow_progress --color=no //:all',
222 | 'Executing Bazel command: bazel clean --color=no',
223 | 'Executing Bazel command: bazel shutdown ',
224 | 'Starting benchmark run 1/2:',
225 | 'Executing Bazel command: bazel build --experimental_generate_json_trace_profile --profile=fake_dir/fake_uid_fake_bazel_commit_0_fake_project_commit_1_of_2.profile.gz --nostamp --noshow_progress --color=no //:all',
226 | 'Executing Bazel command: bazel clean --color=no',
227 | 'Executing Bazel command: bazel shutdown ',
228 | 'Starting benchmark run 2/2:',
229 | 'Executing Bazel command: bazel build --experimental_generate_json_trace_profile --profile=fake_dir/fake_uid_fake_bazel_commit_0_fake_project_commit_2_of_2.profile.gz --nostamp --noshow_progress --color=no //:all',
230 | 'Executing Bazel command: bazel clean --color=no',
231 | 'Executing Bazel command: bazel shutdown '
232 | ]), mock_stderr.getvalue())
233 |
234 |
235 | class BenchmarkFlagsTest(absltest.TestCase):
236 |
237 | @flagsaver.flagsaver
238 | def test_project_source_present(self):
239 | # This mirrors the requirement in benchmark.py
240 | flags.mark_flag_as_required('project_source')
241 | # Assert that the script fails when no project_source is specified
242 | with mock.patch.object(
243 | sys, 'stderr', new=mock_stdio_type()) as mock_stderr, self.assertRaises(
244 | SystemExit) as context:
245 | benchmark.app.run(benchmark.main)
246 | self.assertIn(
247 | ''.join([
248 | 'FATAL Flags parsing error: flag --project_source=None: ',
249 | 'Flag --project_source must have a value other than None.'
250 | ]), mock_stderr.getvalue())
251 |
252 | @flagsaver.flagsaver(bazel_commits=['a', 'b'], project_commits=['c', 'd'])
253 | def test_either_bazel_commits_project_commits_single_element(self):
254 | with self.assertRaises(ValueError) as context:
255 | benchmark._flag_checks()
256 | value_err = context.exception
257 | self.assertEqual(
258 | str(value_err),
259 | 'Either --bazel_commits or --project_commits should be a single element.'
260 | )
261 |
262 | @flagsaver.flagsaver(clean=False)
263 | def test_single_run_skip_clean(self):
264 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr:
265 | benchmark._single_run(
266 | 'bazel_binary_path',
267 | 'build',
268 | options=[],
269 | targets=['//:all'],
270 | startup_options=[])
271 |
272 | self.assertEqual(
273 | ''.join([
274 | 'Executing Bazel command: bazel build --nostamp --noshow_progress --color=no //:all',
275 | 'Executing Bazel command: bazel shutdown '
276 | ]), mock_stderr.getvalue())
277 |
278 | @flagsaver.flagsaver(shutdown=False)
279 | def test_single_run_skip_shutdown(self):
280 | with mock.patch.object(sys, 'stderr', new=mock_stdio_type()) as mock_stderr:
281 | benchmark._single_run(
282 | 'bazel_binary_path',
283 | 'build',
284 | options=[],
285 | targets=['//:all'],
286 | startup_options=[])
287 |
288 | self.assertEqual(
289 | ''.join([
290 | 'Executing Bazel command: bazel build --nostamp --noshow_progress --color=no //:all',
291 | 'Executing Bazel command: bazel clean --color=no'
292 | ]), mock_stderr.getvalue())
293 |
294 |
295 | if __name__ == '__main__':
296 | absltest.main()
297 |
--------------------------------------------------------------------------------
/report/BUILD:
--------------------------------------------------------------------------------
1 | load("@rules_python//python:defs.bzl", "py_binary")
2 | load("@third_party//:requirements.bzl", "requirement")
3 |
4 | package(default_visibility = ["//visibility:public"])
5 |
6 | py_binary(
7 | name = "generate_report",
8 | srcs = ["generate_report.py"],
9 | deps = [
10 | # This is a workaround for https://github.com/bazelbuild/rules_python/issues/14,
11 | # google-cloud-bigquery must be listed first.
12 | requirement("google-cloud-bigquery"),
13 | requirement("cachetools"),
14 | requirement("google-api-core"),
15 | requirement("google-auth"),
16 | requirement("google-cloud-core"),
17 | requirement("google-resumable-media"),
18 | requirement("googleapis-common-protos"),
19 | requirement("protobuf"),
20 | requirement("pytz"),
21 | requirement("requests"),
22 | ],
23 | )
24 |
25 | py_binary(
26 | name = "generate_master_report",
27 | srcs = ["generate_master_report.py"],
28 | deps = [
29 | # This is a workaround for https://github.com/bazelbuild/rules_python/issues/14,
30 | # google-cloud-bigquery must be listed first.
31 | requirement("google-cloud-bigquery"),
32 | requirement("cachetools"),
33 | requirement("google-api-core"),
34 | requirement("google-auth"),
35 | requirement("google-cloud-core"),
36 | requirement("google-resumable-media"),
37 | requirement("googleapis-common-protos"),
38 | requirement("protobuf"),
39 | requirement("pytz"),
40 | requirement("requests"),
41 | ],
42 | )
43 |
--------------------------------------------------------------------------------
/report/generate_master_report.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | #
3 | # Copyright 2019 The Bazel Authors. All rights reserved.
4 | #
5 | # Licensed under the Apache License, Version 2.0 (the "License");
6 | # you may not use this file except in compliance with the License.
7 | # You may obtain a copy of the License at
8 | #
9 | # http:#www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | """Generates a daily HTML report for the projects.
17 |
18 | The steps:
19 | 1. Get the necessary data from Storage for projects/date.
20 | 2. Manipulate the data to a format suitable for graphs.
21 | 3. Generate a HTML report containing the graphs.
22 | 4. Upload the generated HTMLs to GCP Storage.
23 | """
24 | import argparse
25 | import collections
26 | import csv
27 | import datetime
28 | import json
29 | import io
30 | import os
31 | import statistics
32 | import subprocess
33 | import sys
34 | import tempfile
35 | import urllib.request
36 | from google.cloud import bigquery
37 |
38 | TMP = tempfile.gettempdir()
39 | REPORTS_DIRECTORY = os.path.join(TMP, ".bazel_bench", "reports")
40 | PLATFORMS = ["macos", "ubuntu1804"]
41 | PROJECT_SOURCE_TO_NAME = {
42 | "https://github.com/bazelbuild/bazel.git": "bazel",
43 | "https://github.com/tensorflow/tensorflow.git": "tensorflow"
44 | }
45 |
46 |
47 | def _upload_to_storage(src_file_path, storage_bucket, destination_dir):
48 | """Uploads the file from src_file_path to the specified location on Storage."""
49 | args = [
50 | "gsutil", "cp", src_file_path,
51 | "gs://{}/{}".format(storage_bucket, destination_dir)
52 | ]
53 | subprocess.run(args)
54 |
55 |
56 | def _get_storage_url(storage_bucket, dated_subdir):
57 | # In this case, the storage_bucket is a Domain-named bucket.
58 | # https://cloud.google.com/storage/docs/domain-name-verification
59 | return "https://{}/{}".format(storage_bucket, dated_subdir)
60 |
61 |
62 | def _short_hash(commit):
63 | return commit[:7]
64 |
65 |
66 | def _row_component(content):
67 | return """
68 |